William Manley
c7b12a8730
Running `ostree commit --tree=ref=a --tree=dir=b` involves reading the
whole of a into an `OstreeMutableTree` before composing `b` on top. This
is inefficient if a is a complete rootfs and b is just touching one file.
We process O(size of a + size of b) directories rather than
O(number of touched dirs).
This commit makes `ostree commit` more efficient at composing multiple
directories together. With `ostree_mutable_tree_fill_empty_from_dirtree`
we create a lazy `OstreeMutableTree` which only reads the underlying
information from disk when needed. We don't need to read all the
subdirectories just to get the checksum of a tree already checked into the
repo.
This provides great speedups when composing a rootfs out of multiple other
rootfs as we do in our build system. We compose multiple containers
together with:
ostree commit --tree=ref=base-rootfs --tree=ref=container1 --tree=ref=container2
and it is much faster now.
As a test I ran
time ostree --repo=... commit --orphan --tree=ref=big-rootfs --tree=dir=modified_etc
Where modified_etc contained a modified sudoers file under /etc. I used
`strace` to count syscalls and I seperatly took timing measurements. To
test with a cold cache I ran
sync && echo 3 | sudo tee /proc/sys/vm/drop_caches
Results:
| | Before | After |
| -------------------- | ------ | ----- |
| Time (cold cache) | 8.1s | 0.12s |
| Time (warm cache) | 3.7s | 0.08s |
| `openat` calls | 53589 | 246 |
| `fgetxattr` calls | 78916 | 0 |
I'm not sure if this will have some negative interaction with the
`_ostree_repo_commit_modifier_apply` which is short-circuited here. I
think it was disabled for `--tree=ref=` anyway, but I'm not certain. All
the tests pass anyway.
I originally implemented this in terms of the `OstreeRepoFile` APIs, but
it was *way* less efficient, opening and reading many files unnecessarily.
Closes: #1643
Approved by: cgwalters
125 lines
5.0 KiB
C
125 lines
5.0 KiB
C
/*
|
|
* Copyright (C) 2011 Colin Walters <walters@verbum.org>
|
|
*
|
|
* SPDX-License-Identifier: LGPL-2.0+
|
|
*
|
|
* This library is free software; you can redistribute it and/or
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
* License as published by the Free Software Foundation; either
|
|
* version 2 of the License, or (at your option) any later version.
|
|
*
|
|
* This library is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
* Lesser General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
* License along with this library; if not, write to the
|
|
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
* Boston, MA 02111-1307, USA.
|
|
*
|
|
* Author: Colin Walters <walters@verbum.org>
|
|
*/
|
|
|
|
#pragma once
|
|
|
|
#include "ostree-types.h"
|
|
|
|
G_BEGIN_DECLS
|
|
|
|
#define OSTREE_TYPE_MUTABLE_TREE (ostree_mutable_tree_get_type ())
|
|
#define OSTREE_MUTABLE_TREE(o) (G_TYPE_CHECK_INSTANCE_CAST ((o), OSTREE_TYPE_MUTABLE_TREE, OstreeMutableTree))
|
|
#define OSTREE_MUTABLE_TREE_CLASS(k) (G_TYPE_CHECK_CLASS_CAST((k), OSTREE_TYPE_MUTABLE_TREE, OstreeMutableTreeClass))
|
|
#define OSTREE_IS_MUTABLE_TREE(o) (G_TYPE_CHECK_INSTANCE_TYPE ((o), OSTREE_TYPE_MUTABLE_TREE))
|
|
#define OSTREE_IS_MUTABLE_TREE_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE ((k), OSTREE_TYPE_MUTABLE_TREE))
|
|
#define OSTREE_MUTABLE_TREE_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o), OSTREE_TYPE_MUTABLE_TREE, OstreeMutableTreeClass))
|
|
|
|
typedef struct OstreeMutableTreeClass OstreeMutableTreeClass;
|
|
|
|
typedef struct {
|
|
gboolean in_files;
|
|
GHashTableIter iter;
|
|
} OstreeMutableTreeIter;
|
|
|
|
struct OstreeMutableTreeClass
|
|
{
|
|
GObjectClass parent_class;
|
|
};
|
|
|
|
_OSTREE_PUBLIC
|
|
GType ostree_mutable_tree_get_type (void) G_GNUC_CONST;
|
|
|
|
_OSTREE_PUBLIC
|
|
OstreeMutableTree *ostree_mutable_tree_new (void);
|
|
|
|
_OSTREE_PUBLIC
|
|
OstreeMutableTree * ostree_mutable_tree_new_from_checksum (OstreeRepo *repo,
|
|
const char *contents_checksum,
|
|
const char *metadata_checksum);
|
|
|
|
_OSTREE_PUBLIC
|
|
void ostree_mutable_tree_set_metadata_checksum (OstreeMutableTree *self,
|
|
const char *checksum);
|
|
|
|
_OSTREE_PUBLIC
|
|
const char *ostree_mutable_tree_get_metadata_checksum (OstreeMutableTree *self);
|
|
|
|
_OSTREE_PUBLIC
|
|
void ostree_mutable_tree_set_contents_checksum (OstreeMutableTree *self,
|
|
const char *checksum);
|
|
|
|
_OSTREE_PUBLIC
|
|
const char *ostree_mutable_tree_get_contents_checksum (OstreeMutableTree *self);
|
|
|
|
_OSTREE_PUBLIC
|
|
gboolean ostree_mutable_tree_replace_file (OstreeMutableTree *self,
|
|
const char *name,
|
|
const char *checksum,
|
|
GError **error);
|
|
|
|
_OSTREE_PUBLIC
|
|
gboolean ostree_mutable_tree_ensure_dir (OstreeMutableTree *self,
|
|
const char *name,
|
|
OstreeMutableTree **out_subdir,
|
|
GError **error);
|
|
|
|
_OSTREE_PUBLIC
|
|
gboolean ostree_mutable_tree_lookup (OstreeMutableTree *self,
|
|
const char *name,
|
|
char **out_file_checksum,
|
|
OstreeMutableTree **out_subdir,
|
|
GError **error);
|
|
|
|
_OSTREE_PUBLIC
|
|
gboolean
|
|
ostree_mutable_tree_ensure_parent_dirs (OstreeMutableTree *self,
|
|
GPtrArray *split_path,
|
|
const char *metadata_checksum,
|
|
OstreeMutableTree **out_parent,
|
|
GError **error);
|
|
|
|
_OSTREE_PUBLIC
|
|
gboolean ostree_mutable_tree_walk (OstreeMutableTree *self,
|
|
GPtrArray *split_path,
|
|
guint start,
|
|
OstreeMutableTree **out_subdir,
|
|
GError **error);
|
|
|
|
_OSTREE_PUBLIC
|
|
gboolean ostree_mutable_tree_fill_empty_from_dirtree (OstreeMutableTree *self,
|
|
OstreeRepo *repo,
|
|
const char *contents_checksum,
|
|
const char *metadata_checksum);
|
|
|
|
_OSTREE_PUBLIC
|
|
gboolean
|
|
ostree_mutable_tree_check_error (OstreeMutableTree *self,
|
|
GError **error);
|
|
|
|
_OSTREE_PUBLIC
|
|
GHashTable * ostree_mutable_tree_get_subdirs (OstreeMutableTree *self);
|
|
_OSTREE_PUBLIC
|
|
GHashTable * ostree_mutable_tree_get_files (OstreeMutableTree *self);
|
|
|
|
G_END_DECLS
|