packaging/ostree
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Imported Upstream version 2016.3

Browse Source
This commit is contained in:
Simon McVittie committed 2016-03-11 08:08:04 +00:00
2 parents 62a4fef292 3d7098bc5e
commit 67e83bd769
166 files changed
+4714 -1869
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

CONTRIBUTING.md
-121
View File
@@ -1,121 +0,0 @@
Submitting patches
------------------
You can:
1. Send mail to ostree-list@gnome.org, with the patch attached
1. Submit a pull request against https://github.com/GNOME/ostree
1. Attach them to https://bugzilla.gnome.org/
Please look at "git log" and match the commit log style.
Running the test suite
----------------------
Currently, ostree uses https://wiki.gnome.org/GnomeGoals/InstalledTests
To run just ostree's tests:
./configure ... --enable-installed-tests
gnome-desktop-testing-runner -p 0 ostree/
Also, there is a regular:
make check
That runs a different set of tests.
Coding style
------------
Indentation is GNU. Files should start with the appropriate mode lines.
Use GCC `__attribute__((cleanup))` wherever possible. If interacting
with a third party library, try defining local cleanup macros.
Use GError and GCancellable where appropriate.
Prefer returning `gboolean` to signal success/failure, and have output
values as parameters.
Prefer linear control flow inside functions (aside from standard
loops). In other words, avoid "early exits" or use of `goto` besides
`goto out;`.
This is an example of an "early exit":
static gboolean
myfunc (...)
{
gboolean ret = FALSE;
/* some code */
/* some more code */
if (condition)
return FALSE;
/* some more code */
ret = TRUE;
out:
return ret;
}
If you must shortcut, use:
if (condition)
{
ret = TRUE;
goto out;
}
A consequence of this restriction is that you are encouraged to avoid
deep nesting of loops or conditionals. Create internal static helper
functions, particularly inside loops. For example, rather than:
while (condition)
{
/* some code */
if (condition)
{
for (i = 0; i < somevalue; i++)
{
if (condition)
{
/* deeply nested code */
}
/* more nested code */
}
}
}
Instead do this:
static gboolean
helperfunc (..., GError **error)
{
if (condition)
{
/* deeply nested code */
}
/* more nested code */
return ret;
}
while (condition)
{
/* some code */
if (!condition)
continue;
for (i = 0; i < somevalue; i++)
{
if (!helperfunc (..., i, error))
goto out;
}
}
CONTRIBUTING.md
Symlink
+1
View File
@@ -0,0 +1 @@
docs/CONTRIBUTING.md
Makefile-libostree.am
+4 -1
View File
@@ -182,7 +182,10 @@ pkgconfig_DATA += src/libostree/ostree-1.pc
gpgreadme_DATA = src/libostree/README-gpg
gpgreadmedir = $(pkgdatadir)/trusted.gpg.d
EXTRA_DIST += src/libostree/README-gpg src/libostree/bupsplit.h
EXTRA_DIST += src/libostree/README-gpg src/libostree/bupsplit.h \
src/libostree/ostree-enumtypes.h.template \
src/libostree/ostree-enumtypes.c.template \
src/libostree/ostree-deployment-private.h
install-mkdir-remotes-d-hook:
mkdir -p $(DESTDIR)$(sysconfdir)/ostree/remotes.d
Makefile-man.am
+58
View File
@@ -0,0 +1,58 @@
# Makefile for man/
#
# Copyright (C) 2016 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
if ENABLE_MAN
man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1
if BUILDOPT_FUSE
man1_files += rofiles-fuse.1
endif
man5_files = ostree.repo.5 ostree.repo-config.5
man1_MANS = $(addprefix man/,$(man1_files))
man5_MANS = $(addprefix man/,$(man5_files))
EXTRA_DIST += $(man1_MANS) $(man5_MANS)
XSLT_STYLESHEET = http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl
XSLTPROC_FLAGS = \
--nonet \
--stringparam man.output.quietly 1 \
--stringparam funcsynopsis.style ansi \
--stringparam man.th.extra1.suppress 1 \
--stringparam man.authors.section.enabled 0 \
--stringparam man.copyright.section.enabled 0
XSLTPROC_MAN = $(XSLTPROC) $(XSLTPROC_FLAGS)
%.1: %.xml
$(AM_V_GEN) $(XSLTPROC_MAN) --output $@ $(XSLT_STYLESHEET) $<
%.5: %.xml
$(AM_V_GEN) $(XSLTPROC_MAN) --output $@ $(XSLT_STYLESHEET) $<
CLEANFILES += \
$(man1_MANS) \
$(man5_MANS) \
$(NULL)
endif
Makefile-ostree.am
+8
View File
@@ -28,6 +28,7 @@ ostree_SOURCES = src/ostree/main.c \
src/ostree/ot-builtin-checksum.c \
src/ostree/ot-builtin-commit.c \
src/ostree/ot-builtin-diff.c \
src/ostree/ot-builtin-export.c \
src/ostree/ot-builtin-fsck.c \
src/ostree/ot-builtin-gpg-sign.c \
src/ostree/ot-builtin-init.c \
@@ -89,6 +90,8 @@ ostree_SOURCES += \
src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile
$(AM_V_GEN) $(YACC) $< -o $@
EXTRA_DIST += src/ostree/parse-datetime.y
CLEANFILES += src/ostree/parse-datetime.c
ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(srcdir)/src/ostree \
$(NULL)
@@ -105,3 +108,8 @@ ostree_SOURCES += \
ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS)
ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS)
endif
if USE_LIBARCHIVE
ostree_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS)
ostree_LDADD += $(OT_DEP_LIBARCHIVE_LIBS)
endif
Makefile-tests.am
+16
View File
@@ -27,6 +27,7 @@ testfiles = test-basic \
test-remote-add \
test-remote-gpg-import \
test-commit-sign \
test-export \
test-help \
test-libarchive \
test-pull-archive-z \
@@ -61,6 +62,11 @@ testfiles = test-basic \
test-auto-summary \
test-prune \
$(NULL)
if BUILDOPT_FUSE
testfiles += test-rofiles-fuse
endif
insttest_SCRIPTS = $(addprefix tests/,$(testfiles:=.sh))
# This one uses corrupt-repo-ref.js
@@ -79,6 +85,8 @@ insttest_DATA = tests/archive-test.sh \
tests/test-basic-user.sh \
tests/test-local-pull.sh \
tests/corrupt-repo-ref.js \
tests/pre-endian-deltas-repo-big.tar.xz \
tests/pre-endian-deltas-repo-little.tar.xz \
$(NULL)
insttest_SCRIPTS += \
@@ -134,6 +142,10 @@ TESTS = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test-
tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \
tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum
if USE_LIBARCHIVE
TESTS += tests/test-libarchive-import
endif
check_PROGRAMS = $(TESTS)
TESTS_ENVIRONMENT = \
G_TEST_SRCDIR=$(abs_srcdir)/tests \
@@ -166,6 +178,10 @@ tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c
tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags)
tests_test_checksum_LDADD = $(TESTS_LDADD)
tests_test_libarchive_import_SOURCES = tests/test-libarchive-import.c
tests_test_libarchive_import_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags)
tests_test_libarchive_import_LDADD = $(TESTS_LDADD)
tests_test_keyfile_utils_CFLAGS = $(TESTS_CFLAGS)
tests_test_keyfile_utils_LDADD = $(TESTS_LDADD)
Makefile.am
+5 -3
View File
@@ -32,7 +32,7 @@ DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode
SUBDIRS += .
if ENABLE_GTK_DOC
SUBDIRS += doc
SUBDIRS += apidoc
endif
EXTRA_DIST += autogen.sh COPYING README.md
@@ -43,8 +43,8 @@ OT_INTERNAL_SOUP_CFLAGS = $(OT_DEP_SOUP_CFLAGS)
OT_INTERNAL_SOUP_LIBS = $(OT_DEP_SOUP_LIBS)
# This canonicalizes the PKG_CHECK_MODULES or AM_PATH_GPGME results
OT_INTERNAL_GPGME_CFLAGS = $(OT_DEP_GPGME_CFLAGS) $(GPGME_CFLAGS)
OT_INTERNAL_GPGME_LIBS = $(OT_DEP_GPGME_LIBS) $(GPGME_LIBS)
OT_INTERNAL_GPGME_CFLAGS = $(OT_DEP_GPGME_CFLAGS) $(GPGME_PTHREAD_CFLAGS)
OT_INTERNAL_GPGME_LIBS = $(OT_DEP_GPGME_LIBS) $(GPGME_PTHREAD_LIBS)
if BUILDOPT_INTROSPECTION
include $(INTROSPECTION_MAKEFILE)
@@ -68,8 +68,10 @@ include Makefile-otutil.am
include Makefile-libostree.am
include Makefile-ostree.am
include Makefile-switchroot.am
include src/rofiles-fuse/Makefile-inc.am
include Makefile-tests.am
include Makefile-boot.am
include Makefile-man.am
release-tag:
git tag -m "Release $(VERSION)" v$(VERSION)
README.md
+30 -14
View File
@@ -1,31 +1,48 @@
OSTree
======
New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ )
-----
OSTree is a tool that combines a "git-like" model for committing and
downloading bootable filesystem trees, along with a layer for
deploying them and managing the bootloader configuration.
Traditional package managers (dpkg/rpm) build filesystem trees on the
client side. In contrast, the primary focus of OSTree is on
replicating trees composed on a server.
OSTree is like git in that it checksums individual files and has a
content-addressed-object store. It's unlike git in that it "checks
out" the files via hardlinks, and they should thus be immutable.
Therefore, another way to think of OSTree is that it's just a more
polished version of
[Linux VServer hardlinks](http://linux-vserver.org/index.php?title=util-vserver:Vhashify&oldid=2285).
**Features:**
- Atomic upgrades and rollback
- GPG signatures and "pinned TLS" support
- Atomic upgrades and rollback for the system
- Replicating content incrementally over HTTP via GPG signatures and "pinned TLS" support
- Support for parallel installing more than just 2 bootable roots
- Binary history on the server side
- Binary history on the server side (and client)
- Introspectable shared library API for build and deployment systems
This last point is important - you should think of the OSTree command
line as effectively a "demo" for the shared library. The intent is that
package managers, system upgrade tools, container build tools and the like
use OSTree as a "deduplicating hardlink store".
Projects using OSTree
---------------------
[rpm-ostree](https://github.com/projectatomic/rpm-ostree) is a tool
that uses OSTree as a shared library, and supports committing RPMs
into an OSTree repository, and deploying them on the client.
into an OSTree repository, and deploying them on the client. This is
appropriate for "fixed purpose" systems. There is in progress work
for more sophisticated hybrid models, deeply integrating the RPM
packaging with OSTree.
[Project Atomic](http://www.projectatomic.io/) uses rpm-ostree
to provide a minimal host for Docker formatted Linux containers.
[Project Atomic](http://www.projectatomic.io/) uses rpm-ostree to
provide a minimal host for Docker formatted Linux containers.
Replicating a base immutable OS, then using Docker for applications
meshes together two different tools with different tradeoffs.
[xdg-app](https://github.com/alexlarsson/xdg-app) uses OSTree
for desktop application containers.
@@ -46,8 +63,8 @@ versions support extended validation using
However, in order to build from a git clone, you must update the
submodules. If you're packaging OSTree and want a tarball, I
recommend using a "recursive git archive" script. There are several
available online; [this
code](https://git.gnome.org/browse/ostree/tree/packaging/Makefile.dist-packaging#n11)
available online;
[this code](https://git.gnome.org/browse/ostree/tree/packaging/Makefile.dist-packaging#n11)
in OSTree is an example.
Once you have a git clone or recursive archive, building is the
@@ -63,12 +80,11 @@ make install DESTDIR=/path/to/dest
More documentation
------------------
New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ )
Some more information is available on the old wiki page:
https://wiki.gnome.org/Projects/OSTree
The intent is for that wiki page content to be migrated into Markdown
in this git repository.
Contributing
------------
doc/.gitignore → apidoc/.gitignore
View File
File renamed without changes.
doc/Makefile.am → apidoc/Makefile.am
-45
View File
@@ -95,11 +95,6 @@ HTML_IMAGES=
# Extra SGML files that are included by $(DOC_MAIN_SGML_FILE).
# e.g. content_files=running.sgml building.sgml changes-2.0.sgml
content_files= \
overview.xml \
repo.xml \
deployment.xml \
atomic-upgrades.xml \
adapting-existing.xml \
$(NULL)
# SGML files where gtk-doc abbrevations (#GtkWidget) are expanded
@@ -116,53 +111,13 @@ expand_content_files= \
# e.g. GTKDOC_LIBS=$(top_builddir)/gtk/$(gtktargetlib)
GTKDOC_LIBS=
# Hacks around gtk-doc brokenness for out of tree builds
ostree-sections.txt: $(srcdir)/ostree-sections.txt
cp $< $@
version.xml:
echo -n $(VERSION) > "$@"
# This includes the standard gtk-doc make rules, copied by gtkdocize.
include $(top_srcdir)/gtk-doc.make
man1_MANS = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1
man5_MANS = ostree.repo.5 ostree.repo-config.5
if ENABLE_GTK_DOC
XSLTPROC_FLAGS = \
--nonet \
--stringparam man.output.quietly 1 \
--stringparam funcsynopsis.style ansi \
--stringparam man.th.extra1.suppress 1 \
--stringparam man.authors.section.enabled 0 \
--stringparam man.copyright.section.enabled 0
XSLTPROC_MAN = \
$(XSLTPROC) $(XSLTPROC_FLAGS) http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl
.xml.1:
$(AM_V_GEN) $(XSLTPROC_MAN) $<
.xml.5:
$(AM_V_GEN) $(XSLTPROC_MAN) $<
CLEANFILES += \
$(man1_MANS) \
$(man5_MANS) \
$(NULL)
endif # ENABLE_GTK_DOC
MAN_IN_FILES = \
$(man1_MANS:.1=.xml) \
$(man5_MANS:.5=.xml) \
$(NULL)
EXTRA_DIST += \
$(MAN_IN_FILES) \
version.xml \
ostree.xml \
ostree-sections.txt \
$(NULL)
doc/ostree-docs.xml → apidoc/ostree-docs.xml
+1 -7
View File
@@ -7,16 +7,10 @@
]>
<book id="index">
<bookinfo>
<title>OSTree Manual</title>
<title>OSTree API references</title>
<releaseinfo>for OSTree &version;</releaseinfo>
</bookinfo>
<xi:include href="overview.xml"/>
<xi:include href="repo.xml"/>
<xi:include href="deployment.xml"/>
<xi:include href="atomic-upgrades.xml"/>
<xi:include href="adapting-existing.xml"/>
<chapter xml:id="reference">
<title>API Reference</title>
<xi:include href="xml/libostree-core.xml"/>
doc/ostree-sections.txt → apidoc/ostree-sections.txt
+1
View File
@@ -220,6 +220,7 @@ ostree_repo_new_for_sysroot_path
ostree_repo_new_default
ostree_repo_open
ostree_repo_set_disable_fsync
ostree_repo_get_disable_fsync
ostree_repo_is_system
ostree_repo_is_writable
ostree_repo_create
configure.ac
+38 -18
View File
@@ -1,5 +1,5 @@
AC_PREREQ([2.63])
AC_INIT([ostree], [2016.1], [walters@verbum.org])
AC_INIT([ostree], [2016.3], [walters@verbum.org])
AC_CONFIG_HEADER([config.h])
AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_AUX_DIR([build-aux])
@@ -106,17 +106,19 @@ AM_CONDITIONAL(BUILDOPT_INTROSPECTION, test "x$found_introspection" = xyes)
LIBGPGME_DEPENDENCY="1.1.8"
PKG_CHECK_MODULES(OT_DEP_GPGME, gpgme >= $LIBGPGME_DEPENDENCY, have_gpgme=yes, [
m4_ifdef([AM_PATH_GPGME], [
AM_PATH_GPGME($LIBGPGME_DEPENDENCY, have_gpgme=yes, have_gpgme=no)
PKG_CHECK_MODULES(OT_DEP_GPGME, gpgme-pthread >= $LIBGPGME_DEPENDENCY, have_gpgme=yes, [
m4_ifdef([AM_PATH_GPGME_PTHREAD], [
AM_PATH_GPGME_PTHREAD($LIBGPGME_DEPENDENCY, have_gpgme=yes, have_gpgme=no)
],[ have_gpgme=no ])
])
AS_IF([ test x$have_gpgme = xno ], [
AC_MSG_ERROR([Need GPGME version $LIBGPGME_DEPENDENCY or later])
AC_MSG_ERROR([Need GPGME_PTHREAD version $LIBGPGME_DEPENDENCY or later])
])
OSTREE_FEATURES="$OSTREE_FEATURES +gpgme"
LIBARCHIVE_DEPENDENCY="libarchive >= 2.8.0"
# What's in RHEL7.2.
FUSE_DEPENDENCY="fuse >= 2.9.2"
# check for gtk-doc
m4_ifdef([GTK_DOC_CHECK], [
@@ -126,7 +128,22 @@ enable_gtk_doc=no
AM_CONDITIONAL([ENABLE_GTK_DOC], false)
])
AC_PATH_PROG([XSLTPROC], [xsltproc])
AC_ARG_ENABLE(man,
[AS_HELP_STRING([--enable-man],
[generate man pages [default=auto]])],,
enable_man=maybe)
AS_IF([test "$enable_man" != no], [
AC_PATH_PROG([XSLTPROC], [xsltproc])
AS_IF([test -z "$XSLTPROC"], [
AS_IF([test "$enable_man" = yes], [
AC_MSG_ERROR([xsltproc is required for --enable-man])
])
enable_man=no
])
enable_man=yes
])
AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no)
AC_ARG_WITH(libarchive,
AS_HELP_STRING([--without-libarchive], [Do not use libarchive]),
@@ -179,6 +196,16 @@ AS_IF([ test x$with_selinux != xno ], [
if test x$with_selinux != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +selinux"; fi
AM_CONDITIONAL(USE_SELINUX, test $with_selinux != no)
# Enabled by default because I think people should use it.
AC_ARG_ENABLE(rofiles-fuse,
[AS_HELP_STRING([--enable-rofiles-fuse],
[generate rofiles-fuse helper [default=yes]])],,
enable_rofiles_fuse=yes)
AS_IF([ test $enable_rofiles_fuse != xno ], [
PKG_CHECK_MODULES(BUILDOPT_FUSE, $FUSE_DEPENDENCY)
], [enable_rofiles_fuse=no])
AM_CONDITIONAL(BUILDOPT_FUSE, test x$enable_rofiles_fuse = xyes)
AC_ARG_WITH(dracut,
AS_HELP_STRING([--with-dracut],
[Install dracut module (default: no)]),,
@@ -220,18 +247,9 @@ AS_IF([test "x$found_introspection" = xyes], [
], [have_gjs=no])
AM_CONDITIONAL(BUILDOPT_GJS, test x$have_gjs = xyes)
AC_ARG_ENABLE(static_deltas,
AS_HELP_STRING([--enable-static-deltas],
[Enable static delta code (default: yes)]),,
[enable_static_deltas=yes])
AS_IF([test x$enable_static_deltas = xyes], [
AC_DEFINE([BUILDOPT_STATIC_DELTAS], 1, [Define if static deltas are enabled])
])
AM_CONDITIONAL(BUILDOPT_STATIC_DELTAS, test x$enable_static_deltas = xyes)
AC_CONFIG_FILES([
Makefile
doc/Makefile
apidoc/Makefile
src/libostree/ostree-1.pc
])
AC_OUTPUT
@@ -242,12 +260,14 @@ echo "
introspection: $found_introspection
rofiles-fuse: $enable_rofiles_fuse
libsoup (retrieve remote HTTP repositories): $with_soup
libsoup TLS client certs: $have_libsoup_client_certs
SELinux: $with_selinux
libarchive (parse tar files directly): $with_libarchive
static deltas: $enable_static_deltas
documentation: $enable_gtk_doc
static deltas: yes (always enabled now)
man pages (xsltproc): $enable_man
api docs (gtk-doc): $enable_gtk_doc
gjs-based tests: $have_gjs
dracut: $with_dracut
mkinitcpio: $with_mkinitcpio"
doc/adapting-existing.xml
-267
View File
@@ -1,267 +0,0 @@
<?xml version="1.0"?>
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd" [
<!ENTITY version SYSTEM "../version.xml">
]>
<part id="adapting-existing">
<title>Adapting existing mainstream distributions</title>
<chapter id="layout">
<title>System layout</title>
<para>
First, OSTree encourages systems to implement <ulink
url="http://www.freedesktop.org/wiki/Software/systemd/TheCaseForTheUsrMerge/">UsrMove</ulink>.
This is simply to avoid the need for more bind mounts. By
default OSTree's dracut hook creates a read-only bind mount over
<filename class='directory'>/usr</filename>; you can of course
generate individual bind-mounts for <filename
class='directory'>/bin</filename>, all the <filename
class='directory'>/lib</filename> variants, etc. So it is not
intended to be a hard requirement.
</para>
<para>
Remember, because by default the system is booted into a
<literal>chroot</literal> equivalent, there has to be some way
to refer to the actual physical root filesystem. Therefore,
your operating system tree should contain an empty <filename
class='directory'>/sysroot</filename> directory; at boot time,
OSTree will make this a bind mount to the physical / root
directory. There is precedent for this name in the initramfs
context. You should furthermore make a toplevel symbolic link
<filename class='directory'>/ostree</filename> which points to
<filename class='directory'>/sysroot/ostree</filename>, so that
the OSTree tool at runtime can consistently find the system data
regardless of whether it's operating on a physical root or
inside a deployment.
</para>
<para>
Because OSTree only preserves <filename
class='directory'>/var</filename> across upgrades (each
deployment's chroot directory will be garbage collected
eventually), you will need to choose how to handle other
toplevel writable directories specified by the <ulink
url="http://www.pathname.com/fhs/">Filesystem Hierarchy
Standard</ulink>. Your operating system may of course choose
not to support some of these such as <filename
class='directory'>/usr/local</filename>, but following is the
recommended set:
<itemizedlist>
<listitem>
<para>
<filename class='directory'>/home</filename> to <filename class='directory'>/var/home</filename>
</para>
</listitem>
<listitem>
<para>
<filename class='directory'>/opt</filename> to <filename class='directory'>/var/opt</filename>
</para>
</listitem>
<listitem>
<para>
<filename class='directory'>/srv</filename> to <filename class='directory'>/var/srv</filename>
</para>
</listitem>
<listitem>
<para>
<filename class='directory'>/root</filename> to <filename class='directory'>/var/roothome</filename>
</para>
</listitem>
<listitem>
<para>
<filename class='directory'>/usr/local</filename> to <filename class='directory'>/var/local</filename>
</para>
</listitem>
<listitem>
<para>
<filename class='directory'>/mnt</filename> to <filename class='directory'>/var/mnt</filename>
</para>
</listitem>
<listitem>
<para>
<filename class='directory'>/tmp</filename> to <filename class='directory'>/sysroot/tmp</filename>
</para>
</listitem>
</itemizedlist>
</para>
<para>
Furthermore, since <filename class='directory'>/var</filename>
is empty by default, your operating system will need to
dynamically create the <emphasis>targets</emphasis> of these at
boot. A good way to do this is using
<command>systemd-tmpfiles</command>, if your OS uses systemd.
For example:
</para>
<programlisting>
<![CDATA[
d /var/log/journal 0755 root root -
L /var/home - - - - ../sysroot/home
d /var/opt 0755 root root -
d /var/srv 0755 root root -
d /var/roothome 0700 root root -
d /var/usrlocal 0755 root root -
d /var/usrlocal/bin 0755 root root -
d /var/usrlocal/etc 0755 root root -
d /var/usrlocal/games 0755 root root -
d /var/usrlocal/include 0755 root root -
d /var/usrlocal/lib 0755 root root -
d /var/usrlocal/man 0755 root root -
d /var/usrlocal/sbin 0755 root root -
d /var/usrlocal/share 0755 root root -
d /var/usrlocal/src 0755 root root -
d /var/mnt 0755 root root -
d /run/media 0755 root root -
]]>
</programlisting>
<para>
Particularly note here the double indirection of <filename
class='directory'>/home</filename>. By default, each
deployment will share the global toplevel <filename
class='directory'>/home</filename> directory on the physical
root filesystem. It is then up to higher levels of management
tools to keep <filename>/etc/passwd</filename> or equivalent
synchronized between operating systems.
</para>
<para>
Each deployment can easily be reconfigured to have its own home
directory set simply by making <filename
class='directory'>/var/home</filename> a real directory.
</para>
</chapter>
<chapter id="booting">
<title>Booting and initramfs technology</title>
<para>
OSTree comes with optional dracut+systemd integration code that
parses the <literal>ostree=</literal> kernel command line
argument in the initramfs, and then sets up the read-only bind
mount on <filename class='directory'>/usr</filename>, a bind
mount on the deployment's <filename
class='directory'>/sysroot</filename> to the physical <filename
class='directory'>/</filename>, and then finally uses
<literal>mount(MS_MOVE)</literal> to make the deployment root appear to be the
root filesystem before telling systemd to switch root.
</para>
<para>
If you are not using dracut or systemd, using OSTree should still
be possible, but you will have to write the integration code. Patches
to support other initramfs technologies and init systems, if sufficiently
clean, will likely be accepted upstream.
</para>
<para>
A further specific note regarding <command>sysvinit</command>:
OSTree used to support recording device files such the
<filename>/dev/initctl</filename> FIFO, but no longer does.
It's recommended to just patch your initramfs to create this at
boot.
</para>
</chapter>
<chapter id="lib-passwd">
<title>/usr/lib/passwd</title>
<para>
Unlike traditional package systems, OSTree trees contain
<emphasis>numeric</emphasis> uid and gids. Furthermore, it does
not have a <literal>%post</literal> type mechanism where
<filename>useradd</filename> could be invoked. In order to ship
an OS that contains both system users and users dynamically
created on client machines, you will need to choose a solution
for <filename>/etc/passwd</filename>. The core problem is that
if you add a user to the system for a daemon, the OSTree upgrade
process for <filename class='directory'>/etc</filename> will
simply notice that because <filename>/etc/passwd</filename>
differs from the previous default, it will keep the modified
config file, and your new OS user will not be visible.
</para>
<para>
The solution chosen for the <ulink
url="https://live.gnome.org/Projects/GnomeContinuous">gnome-continuous</ulink>
operating system is to create
<filename>/usr/lib/passwd</filename>, and to include a NSS
module <ulink
url="https://github.com/aperezdc/nss-altfiles">nss-altfiles</ulink>
which instructs glibc to read from it. Then, the build system
places all system users there, freeing up
<filename>/etc/passwd</filename> to be purely a database of
local users. See also a more recent effort from <ulink
url="http://0pointer.de/blog/projects/stateless.html">Systemd
stateless</ulink>.
</para>
</chapter>
<chapter id="adapting-package-manager">
<title>Adapting existing package managers</title>
<para>
The largest endeavor is likely to be redesigning your
distribution's package manager to be on top of OSTree,
particularly if you want to keep compatibility with the "old
way" of installing into the physical <filename
class='directory'>/</filename>. This section will use examples
from both <command>dpkg</command> and <command>rpm</command> as
the author has familiarity with both; but the abstract concepts
should apply to most traditional package managers.
</para>
<para>
There are many levels of possible integration; initially, we
will describe the most naive implementation which is the
simplest but also the least efficient. We will assume here that
the admin is booted into an OSTree-enabled system, and wants to
add a set of packages.
</para>
<para>
Many package managers store their state in <filename
class='directory'>/var</filename>; but since in the OSTree model
that directory is shared between independent versions, the
package database must first be found in the per-deployment
<filename class='directory'>/usr</filename> directory. It
becomes read-only; remember, all upgrades involve constructing a
new filesystem tree, so your package manager will also need to
create a copy of its database. Most likely, if you want to
continue supporting non-OSTree deployments, simply have your
package manager fall back to the legacy <filename
class='directory'>/var</filename> location if the one in
<filename class='directory'>/usr</filename> is not found.
</para>
<para>
To install a set of new packages (without removing any existing
ones), enumerate the set of packages in the currently booted
deployment, and perform dependency resolution to compute the
complete set of new packages. Download and unpack these new
packages to a temporary directory.
</para>
<para>
Now, because we are merely installing new packages and not
removing anything, we can make the major optimization of reusing
our existing filesystem tree, and merely
<emphasis>layering</emphasis> the composed filesystem tree of
these new packages on top. A command lke this: <command>ostree
commit -b osname/releasename/description
--tree=ref=<replaceable>osname/releasenamename/description</replaceable>
--tree=dir=/var/tmp/newpackages.13A8D0/</command> will create a
new commit in the
<replaceable>osname/releasename/description</replaceable>
branch. The OSTree SHA256 checksum of all the files in
/var/tmp/newpackages.13A8D0/ will be computed, but we will not
re-checksum the present existing tree. In this layering model,
earlier directories will take precedence, but files in later
layers will silently override earlier layers.
</para>
<para>
Then to actually deploy this tree for the next boot:
<command>ostree admin deploy
<replaceable>osname/releasenamename/description</replaceable></command>
</para>
</chapter>
</part>
doc/atomic-upgrades.xml
-181
View File
@@ -1,181 +0,0 @@
<?xml version="1.0"?>
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd" [
<!ENTITY version SYSTEM "../version.xml">
]>
<part id="atomic-upgrades">
<title>Atomic Upgrades</title>
<chapter id="upgrades-intro">
<title>You can turn off the power anytime you want...</title>
<para>
OSTree is designed to implement fully atomic and safe upgrades;
more generally, atomic transitions between lists of bootable
deployments. If the system crashes or you pull the power, you
will have either the old system, or the new one.
</para>
</chapter>
<chapter id="simple-http">
<title>Simple upgrades via HTTP</title>
<para>
First, the most basic model OSTree supports is one where it
replicates pre-generated filesystem trees from a server over
HTTP, tracking exactly one ref, which is stored in the <filename
class='extension'>.origin</filename> file for the deployment.
The command <command>ostree admin upgrade</command> implements
this.
</para>
<para>
To begin a simple upgrade, OSTree fetches the contents of the
ref from the remote server. Suppose we're tracking a ref named
<literal>exampleos/buildmaster/x86_64-runtime</literal>.
OSTree fetches the URL
<literal>http://<replaceable>example.com</replaceable>/repo/refs/exampleos/buildmaster/x86_64-runtime</literal>,
which contains a SHA256 checksum. This determines the tree to
deploy, and <filename class='directory'>/etc</filename> will be
merged from currently booted tree.
</para>
<para>
If we do not have this commit, then, then we perform a pull
process. At present (without static deltas), this involves
quite simply just fetching each individual object that we do not
have, asynchronously. Put in other words, we only download
changed files (zlib-compressed). Each object has its checksum
validated and is stored in <filename
class='directory'>/ostree/repo/objects/</filename>.
</para>
<para>
Once the pull is complete, we have all the objects locally
we need to perform a deployment.
</para>
</chapter>
<chapter id="package-manager">
<title>Upgrades via external tools (e.g. package managers)</title>
<para>
As mentioned in the introduction, OSTree is also designed to
allow a model where filesystem trees are computed on the client.
It is completely agnostic as to how those trees are generated;
they could be computed with traditional packages, packages with
post-deployment scripts on top, or built by developers directly
from revision control locally, etc.
</para>
<para>
At a practical level, most package managers today
(<command>dpkg</command> and <command>rpm</command>) operate
"live" on the currently booted filesystem. The way they could
work with OSTree is instead to take the list of installed
packages in the currently booted tree, and compute a new
filesystem from that. A later chapter describes in more details
how this could work: <xref linkend="adapting-existing"/>.
</para>
<para>
For the purposes of this section, let's assume that we have a
newly generated filesystem tree stored in the repo (which shares
storage with the existing booted tree). We can then move on to
checking it back out of the repo into a deployment.
</para>
</chapter>
<chapter id="deployment-dir">
<title>Assembling a new deployment directory</title>
<para>
Given a commit to deploy, OSTree first allocates a directory for
it. This is of the form <filename
class='directory'>/boot/loader/entries/ostree-<replaceable>osname</replaceable>-<replaceable>checksum</replaceable>.<replaceable>serial</replaceable>.conf</filename>.
The <replaceable>serial</replaceable> is normally 0, but if a
given commit is deployed more than once, it will be incremented.
This is supported because the previous deployment may have
configuration in <filename class='directory'>/etc</filename>
that we do not want to use or overwrite.
</para>
<para>
Now that we have a deployment directory, a 3-way merge is
performed between the (by default) currently booted deployment's
<filename class='directory'>/etc</filename>, its default
configuration, and the new deployment (based on its <filename
class='directory'>/usr/etc</filename>).
</para>
</chapter>
<chapter id="swapping-boot">
<title>Atomically swapping boot configuration</title>
<para>
At this point, a new deployment directory has been created as a
hardlink farm; the running system is untouched, and the
bootloader configuration is untouched. We want to add this deployment
to the "deployment list".
</para>
<para>
To support a more general case, OSTree supports atomic
transitioning between arbitrary sets of deployments, with the
restriction that the currently booted deployment must always be
in the new set. In the normal case, we have exactly one
deployment, which is the booted one, and we want to add the new
deployment to the list. A more complex command might allow
creating 100 deployments as part of one atomic transaction, so
that one can set up an automated system to bisect across them.
</para>
<simplesect id="bootversion">
<title>The bootversion</title>
<para>
OSTree allows swapping between boot configurations by
implementing the "swapped directory pattern" in <filename
class='directory'>/boot</filename>. This means it is a
symbolic link to one of two directories <filename
class='directory'>/ostree/boot.<replaceable>[0|1]</replaceable></filename>.
To swap the contents atomically, if the current version is
<literal>0</literal>, we create <filename
class='directory'>/ostree/boot.1</filename>, populate it with
the new contents, then atomically swap the symbolic link. Finally,
the old contents can be garbage collected at any point.
</para>
</simplesect>
<simplesect id="ostree-bootversion">
<title>The /ostree/boot directory</title>
<para>
However, we want to optimize for the case where we the set of
kernel/initramfs pairs is the same between both the old and
new deployment lists. This happens when doing an upgrade that
does not include the kernel; think of a simple translation
update. OSTree optimizes for this case because on some
systems <filename class='directory'>/boot</filename> may be on
a separate medium such as flash storage not optimized for
significant amounts of write traffic.
</para>
<para>
To implement this, OSTree also maintains the directory
<filename
class='directory'>/ostree/boot.<replaceable>bootversion</replaceable></filename>,
which is a set of symbolic links to the deployment
directories. The <replaceable>bootversion</replaceable> here
must match the version of <filename
class='directory'>/boot</filename>. However, in order to
allow atomic transitions of <emphasis>this</emphasis>
directory, this is also a swapped directory, so just like
<filename class='directory'>/boot</filename>, it has a version
of <literal>0</literal> or <literal>1</literal> appended.
</para>
<para>
Each bootloader entry has a special <literal>ostree=</literal>
argument which refers to one of these symbolic links. This is
parsed at runtime in the initramfs.
</para>
</simplesect>
</chapter>
</part>
doc/deployment.xml
-158
View File
@@ -1,158 +0,0 @@
<?xml version="1.0"?>
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd" [
<!ENTITY version SYSTEM "../version.xml">
]>
<part id="deployment">
<title>Deployments</title>
<chapter id="deployment-intro">
<title>Overview</title>
<para>
Built on top of the OSTree versioning filesystem core is a layer
that knows how to deploy, parallel install, and manage Unix-like
operating systems (accessible via <command>ostree
admin</command>). The core content of these operating systems
are treated as read-only, but they transparently share storage.
</para>
<para>
A deployment is physically located at a path of the form
<filename
class='directory'>/ostree/deploy/<replaceable>osname</replaceable>/deploy/<replaceable>checksum</replaceable></filename>.
OSTree is designed to boot directly into exactly one deployment
at a time; each deployment is intended to be a target for
<literal>chroot()</literal> or equivalent.
</para>
</chapter>
<chapter id="deployment-osname">
<title>"osname": Group of deployments that share /var</title>
<para>
Each deployment is grouped in exactly one "osname". From
above, you can see that an osname is physically represented in
the <filename
class='directory'>/ostree/deploy/<replaceable>osname</replaceable></filename>
directory. For example, OSTree can allow parallel installing
Debian in <filename
class='directory'>/ostree/deploy/debian</filename> and Red Hat
Enterprise Linux in <filename
class='directory'>/ostree/deploy/rhel</filename> (subject to
operating system support, present released versions of these
operating systems may not support this).
</para>
<para>
Each osname has exactly one copy of the traditional Unix
<filename class='directory'>/var</filename>, stored physically
in <filename
class='directory'>/ostree/deploy/<replaceable>osname</replaceable>/var</filename>.
OSTree provides support tools for <command>systemd</command>
to create a Linux bind mount that ensures the booted
deployment sees the shared copy of <filename
class='directory'>/var</filename>.
</para>
<para>
OSTree does not touch the contents of <filename
class='directory'>/var</filename>. Operating system components
such as daemon services are required to create any directories
they require there at runtime (e.g. <filename
class='directory'>/var/cache/<replaceable>daemonname</replaceable></filename>),
and to manage upgrading data formats inside those directories.
</para>
</chapter>
<chapter id="deployment-contents">
<title>Contents of a deployment</title>
<para>
A deployment begins with a specific commit (represented as a
SHA256 hash) in the OSTree repository in <filename
class='directory'>/ostree/repo</filename>. This commit refers
to a filesystem tree that represents the underlying basis of a
deployment. For short, we will call this the "tree", to
distinguish it from the concept of a deployment.
</para>
<para>
First, the tree must include a kernel stored as <filename
class='directory'>/boot/vmlinuz-<replaceable>checksum</replaceable></filename>.
The checksum should be a SHA256 hash of the kernel contents;
it must be pre-computed before storing the kernel in the
repository. Optionally, the tree can contain an initramfs,
stored as <filename
class='directory'>/boot/initramfs-<replaceable>checksum</replaceable></filename>.
If this exists, the checksum must include both the kernel and
initramfs contents. OSTree will use this to determine which
kernels are shared. The rationale for this is to avoid
computing checksums on the client by default.
</para>
<para>
The deployment should not have a traditional UNIX <filename
class='directory'>/etc</filename>; instead, it should include
<filename class='directory'>/usr/etc</filename>. This is the
"default configuration". When OSTree creates a deployment, it
performs a 3-way merge using the <emphasis>old</emphasis>
default configuration, the active system's <filename
class='directory'>/etc</filename>, and the new default
configuration. In the final filesystem tree for a deployment
then, <filename class='directory'>/etc</filename> is a regular
writable directory.
</para>
<para>
Besides the exceptions of <filename
class='directory'>/var</filename> and <filename
class='directory'>/etc</filename> then, the rest of the
contents of the tree are checked out as hard links into the
repository. It's strongly recommended that operating systems
ship all of their content in <filename
class='directory'>/usr</filename>, but this is not a hard
requirement.
</para>
<para>
Finally, a deployment may have a <filename
class='extension'>.origin</filename> file, stored next to its
directory. This file tells <command>ostree admin
upgrade</command> how to upgrade it. At the moment, OSTree only
supports upgrading a single refspec. However, in the future
OSTree may support a syntax for composing layers of trees, for
example.
</para>
</chapter>
<chapter id="managing-boot">
<title>The system /boot</title>
<para>
While OSTree parallel installs deployments cleanly inside the
<filename class='directory'>/ostree</filename> directory,
ultimately it has to control the system's <filename
class='directory'>/boot</filename> directory. The way this
works is via the <ulink
url="http://www.freedesktop.org/wiki/Specifications/BootLoaderSpec/">boot
loader specification</ulink>, which is a standard for
bootloader-independent drop-in configuration files.
</para>
<para>
When a tree is deployed, it will have a configuration file
generated of the form <filename
class='directory'>/boot/loader/entries/ostree-<replaceable>osname</replaceable>-<replaceable>checksum</replaceable>.<replaceable>serial</replaceable>.conf</filename>.
This configuration file will include a special
<literal>ostree=</literal> kernel argument that allows the
initramfs to find (and <literal>chroot()</literal> into) the
specified deployment.
</para>
<para>
At present, not all bootloaders implement the BootLoaderSpec,
so OSTree contains code for some of these to regenerate native
config files (such as
<filename>/boot/syslinux/syslinux.conf</filename> based on the
entries.
</para>
</chapter>
</part>
doc/overview.xml
-155
View File
@@ -1,155 +0,0 @@
<?xml version="1.0"?>
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd" [
<!ENTITY version SYSTEM "../version.xml">
]>
<part id="overview">
<title>OSTree Overview</title>
<chapter id="ostree-intro">
<title>Introduction</title>
<para>
OSTree an upgrade system for Linux-based operating systems that
performs atomic upgrades of complete filesystem trees. It is
not a package system; rather, it is intended to complement them.
A primary model is composing packages on a server, and then
replicating them to clients.
</para>
<para>
The underlying architecture might be summarized as "git for
operating system binaries". It operates in userspace, and will
work on top of any Linux filesystem. At its core is a git-like
content-addressed object store, and layered on top of that is
bootloader configuration, management of
<filename>/etc</filename>, and other functions to perform an
upgrade beyond just replicating files.
</para>
<para>
You can use OSTree standalone in the pure replication model,
but another approach is to add a package manager on top,
thus creating a hybrid tree/package system.
</para>
</chapter>
<chapter id="ostree-package-comparison">
<title>Comparison with "package managers"</title>
<para>
Because OSTree is designed for deploying core operating
systems, a comparison with traditional "package managers" such
as dpkg and rpm is illustrative. Packages are traditionally
composed of partial filesystem trees with metadata and scripts
attached, and these are dynamically assembled on the client
machine, after a process of dependency resolution.
</para>
<para>
In contrast, OSTree only supports recording and deploying
<emphasis>complete</emphasis> (bootable) filesystem trees. It
has no built-in knowledge of how a given filesystem tree was
generated or the origin of individual files, or dependencies,
descriptions of individual components. Put another way, OSTree
only handles delivery and deployment; you will likely still want
to include inside each tree metadata about the individual
components that went into the tree. For example, a system
administrator may want to know what version of OpenSSL was
included in your tree, so you should support the equivalent of
<command>rpm -q</command> or <command>dpkg -L</command>.
</para>
<para>
The OSTree core emphasizes replicating read-only OS trees via
HTTP, and where the OS includes (if desired) an entirely
separate mechanism to install applications, stored in <filename
class='directory'>/var</filename> if they're system global, or
<filename class='directory'>/home</filename> for per-user
application installation. An example application mechanism is
<ulink url="http://docker.io/">Docker</ulink>.
</para>
<para>
However, it is entirely possible to use OSTree underneath a
package system, where the contents of <filename
class='directory'>/usr</filename> are computed on the client.
For example, when installing a package, rather than changing the
currently running filesystem, the package manager could assemble
a new filesystem tree that layers the new packages on top of a
base tree, record it in the local OSTree repository, and then
set it up for the next boot. To support this model, OSTree
provides an (introspectable) C shared library.
</para>
</chapter>
<chapter id="ostree-block-comparison">
<title>Comparison with block/image replication</title>
<para>
OSTree shares some similarity with "dumb" replication and
stateless deployments, such as the model common in "cloud"
deployments where nodes are booted from an (effectively)
readonly disk, and user data is kept on a different volumes.
The advantage of "dumb" replication, shared by both OSTree and
the cloud model, is that it's <emphasis>reliable</emphasis>
and <emphasis>predictable</emphasis>.
</para>
<para>
But unlike many default image-based deployments, OSTree supports
exactly two persistent writable directories that are preserved
across upgrades: <literal>/etc</literal> and
<literal>/var</literal>.
</para>
<para>
Because OSTree operates at the Unix filesystem layer, it works
on top of any filesystem or block storage layout; it's possible
to replicate a given filesystem tree from an OSTree repository
into plain ext4, BTRFS, XFS, or in general any Unix-compatible
filesystem that supports hard links. Note: OSTree will
transparently take advantage of some BTRFS features if deployed
on it.
</para>
</chapter>
<chapter id="ostree-atomic-parallel-installation">
<title>Atomic transitions between parallel-installable read-only filesystem trees</title>
<para>
Another deeply fundamental difference between both package
managers and image-based replication is that OSTree is
designed to parallel-install <emphasis>multiple
versions</emphasis> of multiple
<emphasis>independent</emphasis> operating systems. OSTree
relies on a new toplevel <filename
class='directory'>ostree</filename> directory; it can in fact
parallel install inside an existing OS or distribution
occupying the physical <filename
class='directory'>/</filename> root.
</para>
<para>
On each client machine, there is an OSTree repository stored
in <filename class='directory'>/ostree/repo</filename>, and a
set of "deployments" stored in <filename
class='directory'>/ostree/deploy/<replaceable>OSNAME</replaceable>/<replaceable>CHECKSUM</replaceable></filename>.
Each deployment is primarily composed of a set of hardlinks
into the repository. This means each version is deduplicated;
an upgrade process only costs disk space proportional to the
new files, plus some constant overhead.
</para>
<para>
The model OSTree emphasizes is that the OS read-only content
is kept in the classic Unix <filename
class='directory'>/usr</filename>; it comes with code to
create a Linux read-only bind mount to prevent inadvertent
corruption. There is exactly one <filename
class='directory'>/var</filename> writable directory shared
between each deployment for a given OS. The OSTree core code
does not touch content in this directory; it is up to the code
in each operating system for how to manage and upgrade state.
</para>
<para>
Finally, each deployment has its own writable copy of the
configuration store <filename
class='directory'>/etc</filename>. On upgrade, OSTree will
perform a basic 3-way diff, and apply any local changes to the
new copy, while leaving the old untouched.
</para>
</chapter>
</part>
doc/repo.xml
-127
View File
@@ -1,127 +0,0 @@
<?xml version="1.0"?>
<!DOCTYPE book PUBLIC "-//OASIS//DTD DocBook XML V4.1.2//EN"
"http://www.oasis-open.org/docbook/xml/4.1.2/docbookx.dtd" [
<!ENTITY version SYSTEM "../version.xml">
]>
<part id="repository">
<title>Anatomy of an OSTree repository</title>
<chapter id="repository-intro">
<title>Core object types and data model</title>
<para>
OSTree is deeply inspired by git; the core layer is a userspace
content-addressed versioning filesystem. It is worth taking
some time to familiarize yourself with <ulink
url="http://git-scm.com/book/en/Git-Internals">Git
Internals</ulink>, as this section will assume some knowledge of
how git works.
</para>
<para>
Its object types are similar to git; it has commit objects and
content objects. Git has "tree" objects, whereas OSTree splits
them into "dirtree" and "dirmeta" objects. But unlike git,
OSTree's checksums are SHA256. And most crucially, its content
objects include uid, gid, and extended attributes (but still no
timestamps).
</para>
<simplesect id="commits">
<title>Commit objects</title>
<para>
A commit object contains metadata such as a timestamp, a log
message, and most importantly, a reference to a
dirtree/dirmeta pair of checksums which describe the root
directory of the filesystem.
</para>
<para>
Also like git, each commit in OSTree can have a parent. It is
designed to store a history of your binary builds, just like git
stores a history of source control. However, OSTree also makes
it easy to delete data, under the assumption that you can
regenerate it from source code.
</para>
</simplesect>
<simplesect id="dirtree">
<title>Dirtree objects</title>
<para>
A dirtree contains a sorted array of (filename, checksum)
pairs for content objects, and a second sorted array of
(filename, dirtree checksum, dirmeta checksum), which are
subdirectories.
</para>
</simplesect>
<simplesect id="dirmeta">
<title>Dirmeta objects</title>
<para>
In git, tree objects contain the metadata such as permissions
for their children. But OSTree splits this into a separate
object to avoid duplicating extended attribute listings.
</para>
</simplesect>
<simplesect id="content">
<title>Content objects</title>
<para>
Unlike the first three object types which are metadata,
designed to be <literal>mmap()ed</literal>, the content object
has a separate internal header and payload sections. The
header contains uid, gid, mode, and symbolic link target (for
symlinks), as well as extended attributes. After the header,
for regular files, the content follows.
</para>
</simplesect>
</chapter>
<chapter id="repository-types">
<title>Repository types and locations</title>
<para>
Also unlike git, an OSTree repository can be in one of two
separate modes: <literal>bare</literal> and
<literal>archive-z2</literal>. A bare repository is one where
content files are just stored as regular files; it's designed to
be the source of a "hardlink farm", where each operating system
checkout is merely links into it. If you want to store files
owned by e.g. root in this mode, you must run OSTree as root.
In contrast, the <literal>archive-z2</literal> mode is designed
for serving via plain HTTP. Like tar files, it can be
read/written by non-root users.
</para>
<para>
On an OSTree-deployed system, the "system repository" is
<filename class='directory'>/ostree/repo</filename>. It can be
read by any uid, but only written by root. Unless the
<literal>--repo</literal> argument is given to the
<command>ostree</command> command, it will operate on the system
repository.
</para>
</chapter>
<chapter id="refs">
<title>Refs</title>
<para>
Like git, OSTree uses "refs" to which are text files that point
to particular commits (i.e. filesystem trees). For example, the
gnome-ostree operating system creates trees named like
<literal>gnome-ostree/buildmaster/x86_64-runtime</literal> and
<literal>gnome-ostree/buildmaster/x86_64-devel-debug</literal>.
These two refs point to two different generated filesystem
trees. In this example, the "runtime" tree contains just enough
to run a basic GNOME system, and "devel-debug" contains all of
the developer tools.
</para>
<para>
The <command>ostree</command> supports a simple syntax using the
carat <literal>^</literal> to refer to the parent of a given
commit. For example,
<literal>gnome-ostree/buildmaster/x86_64-runtime^</literal>
refers to the previous build, and
<literal>gnome-ostree/buildmaster/x86_64-runtime^^</literal>
refers to the one before that.
</para>
</chapter>
</part>
docs-md/atomic-upgrades.md
-12
View File
@@ -1,12 +0,0 @@
Atomic upgrades and rollback
----------------------------
Traditional package managers operate "live" on the running system.
This means it's possible for interrupted updates to result in a
half-updated system. This model also makes it significantly harder to
support rollbacks when updates fail.
In contrast, OSTree always creates a *new* root whenever it's
performing an update. This new root shares storage via hardlinks with
the current system. Upon success, the bootloader configuration will
be updated.
docs/CONTRIBUTING.md
+121
View File
@@ -0,0 +1,121 @@
Submitting patches
------------------
You can:
1. Send mail to ostree-list@gnome.org, with the patch attached
1. Submit a pull request against https://github.com/GNOME/ostree
1. Attach them to https://bugzilla.gnome.org/
Please look at "git log" and match the commit log style.
Running the test suite
----------------------
Currently, ostree uses https://wiki.gnome.org/GnomeGoals/InstalledTests
To run just ostree's tests:
./configure ... --enable-installed-tests
gnome-desktop-testing-runner -p 0 ostree/
Also, there is a regular:
make check
That runs a different set of tests.
Coding style
------------
Indentation is GNU. Files should start with the appropriate mode lines.
Use GCC `__attribute__((cleanup))` wherever possible. If interacting
with a third party library, try defining local cleanup macros.
Use GError and GCancellable where appropriate.
Prefer returning `gboolean` to signal success/failure, and have output
values as parameters.
Prefer linear control flow inside functions (aside from standard
loops). In other words, avoid "early exits" or use of `goto` besides
`goto out;`.
This is an example of an "early exit":
static gboolean
myfunc (...)
{
gboolean ret = FALSE;
/* some code */
/* some more code */
if (condition)
return FALSE;
/* some more code */
ret = TRUE;
out:
return ret;
}
If you must shortcut, use:
if (condition)
{
ret = TRUE;
goto out;
}
A consequence of this restriction is that you are encouraged to avoid
deep nesting of loops or conditionals. Create internal static helper
functions, particularly inside loops. For example, rather than:
while (condition)
{
/* some code */
if (condition)
{
for (i = 0; i < somevalue; i++)
{
if (condition)
{
/* deeply nested code */
}
/* more nested code */
}
}
}
Instead do this:
static gboolean
helperfunc (..., GError **error)
{
if (condition)
{
/* deeply nested code */
}
/* more nested code */
return ret;
}
while (condition)
{
/* some code */
if (!condition)
continue;
for (i = 0; i < somevalue; i++)
{
if (!helperfunc (..., i, error))
goto out;
}
}
docs/index.md
Symlink
+1
View File
@@ -0,0 +1 @@
../README.md
docs/manual/adapting-existing.md
+159
View File
@@ -0,0 +1,159 @@
# Adapting existing mainstream distributions
## System layout
First, OSTree encourages systems to implement
[UsrMove](http://www.freedesktop.org/wiki/Software/systemd/TheCaseForTheUsrMerge/)
This is simply to avoid the need for more bind mounts. By default
OSTree's dracut hook creates a read-only bind mount over `/usr`; you
can of course generate individual bind-mounts for `/bin`, all the
`/lib` variants, etc. So it is not intended to be a hard requirement.
Remember, because by default the system is booted into a `chroot`
equivalent, there has to be some way to refer to the actual physical
root filesystem. Therefore, your operating system tree should contain
an empty `/sysroot` directory; at boot time, OSTree will make this a
bind mount to the physical / root directory. There is precedent for
this name in the initramfs context. You should furthermore make a
toplevel symbolic link `/ostree` which points to `/sysroot/ostree`, so
that the OSTree tool at runtime can consistently find the system data
regardless of whether it's operating on a physical root or inside a
deployment.
Because OSTree only preserves `/var` across upgrades (each
deployment's chroot directory will be garbage collected
eventually), you will need to choose how to handle other
toplevel writable directories specified by the [Filesystem Hierarchy Standard](http://www.pathname.com/fhs/")
Your operating system may of course choose
not to support some of these such as `/usr/local`, but following is the
recommended set:
- `/home` → `/var/home`
- `/opt` → `/var/opt`
- `/srv` → `/var/srv`
- `/root` → `/var/roothome`
- `/usr/local` → `/var/local`
- `/mnt` → `/var/mnt`
- `/tmp` → `/sysroot/tmp`
Furthermore, since `/var` is empty by default, your operating system
will need to dynamically create the <emphasis>targets</emphasis> of
these at boot. A good way to do this is using `systemd-tmpfiles`, if
your OS uses systemd. For example:
```
d /var/log/journal 0755 root root -
L /var/home - - - - ../sysroot/home
d /var/opt 0755 root root -
d /var/srv 0755 root root -
d /var/roothome 0700 root root -
d /var/usrlocal 0755 root root -
d /var/usrlocal/bin 0755 root root -
d /var/usrlocal/etc 0755 root root -
d /var/usrlocal/games 0755 root root -
d /var/usrlocal/include 0755 root root -
d /var/usrlocal/lib 0755 root root -
d /var/usrlocal/man 0755 root root -
d /var/usrlocal/sbin 0755 root root -
d /var/usrlocal/share 0755 root root -
d /var/usrlocal/src 0755 root root -
d /var/mnt 0755 root root -
d /run/media 0755 root root -
```
Particularly note here the double indirection of `/home`. By default,
each deployment will share the global toplevel `/home` directory on
the physical root filesystem. It is then up to higher levels of
management tools to keep <filename>/etc/passwd</filename> or
equivalent synchronized between operating systems. Each deployment
can easily be reconfigured to have its own home directory set simply
by making `/var/home` a real directory. </chapter>
## Booting and initramfs technology
OSTree comes with optional dracut+systemd integration code that parses
the `ostree=` kernel command line argument in the initramfs, and then
sets up the read-only bind mount on `/usr`, a bind mount on the
deployment's `/sysroot` to the physical `/`, and then finally uses
`mount(MS_MOVE)` to make the deployment root appear to be the root
filesystem before telling systemd to switch root.
If you are not using dracut or systemd, using OSTree should still be
possible, but you will have to write the integration code. Patches to
support other initramfs technologies and init systems, if sufficiently
clean, will likely be accepted upstream.
A further specific note regarding `sysvinit`: OSTree used to support
recording device files such the `/dev/initctl` FIFO, but no longer
does. It's recommended to just patch your initramfs to create this at
boot.
## /usr/lib/passwd
Unlike traditional package systems, OSTree trees contain *numeric* uid
and gids. Furthermore, it does not have a `%post` type mechanism
where `useradd` could be invoked. In order to ship an OS that
contains both system users and users dynamically created on client
machines, you will need to choose a solution for `/etc/passwd`. The
core problem is that if you add a user to the system for a daemon, the
OSTree upgrade process for `/etc` will simply notice that because
`/etc/passwd` differs from the previous default, it will keep the
modified config file, and your new OS user will not be visible. The
solution chosen for the [Gnome Continuous](https://live.gnome.org/Projects/GnomeContinuous) operating
system is to create `/usr/lib/passwd`, and to include a NSS module
[nss-altfiles](https://github.com/aperezdc/nss-altfiles) which
instructs glibc to read from it. Then, the build system places all
system users there, freeing up `/etc/passwd` to be purely a database
of local users. See also a more recent effort from [Systemd Stateless](http://0pointer.de/blog/projects/stateless.html)
## Adapting existing package managers
The largest endeavor is likely to be redesigning your distribution's
package manager to be on top of OSTree, particularly if you want to
keep compatibility with the "old way" of installing into the physical
`/`. This section will use examples from both `dpkg` and `rpm` as the
author has familiarity with both; but the abstract concepts should
apply to most traditional package managers.
There are many levels of possible integration; initially, we will
describe the most naive implementation which is the simplest but also
the least efficient. We will assume here that the admin is booted
into an OSTree-enabled system, and wants to add a set of packages.
Many package managers store their state in `/var`; but since in the
OSTree model that directory is shared between independent versions,
the package database must first be found in the per-deployment `/usr`
directory. It becomes read-only; remember, all upgrades involve
constructing a new filesystem tree, so your package manager will also
need to create a copy of its database. Most likely, if you want to
continue supporting non-OSTree deployments, simply have your package
manager fall back to the legacy `/var` location if the one in `/usr`
is not found.
To install a set of new packages (without removing any existing ones),
enumerate the set of packages in the currently booted deployment, and
perform dependency resolution to compute the complete set of new
packages. Download and unpack these new packages to a temporary
directory.
Now, because we are merely installing new packages and not
removing anything, we can make the major optimization of reusing
our existing filesystem tree, and merely
*layering* the composed filesystem tree of
these new packages on top. A command like this:
```
ostree commit -b osname/releasename/description
--tree=ref=$osname/releasename/description
--tree=dir=/var/tmp/newpackages.13A8D0/
```
will create a new commit in the `$osname/releasename/description`
branch. The OSTree SHA256 checksum of all the files in
`/var/tmp/newpackages.13A8D0/` will be computed, but we will not
re-checksum the present existing tree. In this layering model,
earlier directories will take precedence, but files in later layers
will silently override earlier layers.
Then to actually deploy this tree for the next boot:
`ostree admin deploy <replaceable>osname/releasename/description`
docs/manual/atomic-upgrades.md
+116
View File
@@ -0,0 +1,116 @@
# Atomic Upgrades
## You can turn off the power anytime you want...
OSTree is designed to implement fully atomic and safe upgrades;
more generally, atomic transitions between lists of bootable
deployments. If the system crashes or you pull the power, you
will have either the old system, or the new one.
## Simple upgrades via HTTP
First, the most basic model OSTree supports is one where it replicates
pre-generated filesystem trees from a server over HTTP, tracking
exactly one ref, which is stored in the `.origin` file for the
deployment. The command `ostree admin upgrade`
implements this.
o begin a simple upgrade, OSTree fetches the contents of the ref from
the remote server. Suppose we're tracking a ref named
`exampleos/buildmaster/x86_64-runtime`. OSTree fetches the URL
`http://$example.com/repo/refs/exampleos/buildmaster/x86_64-runtime`,
which contains a SHA256 checksum. This determines the tree to deploy,
and `/etc` will be merged from currently booted tree.
If we do not have this commit, then, then we perform a pull process.
At present (without static deltas), this involves quite simply just
fetching each individual object that we do not have, asynchronously.
Put in other words, we only download changed files (zlib-compressed).
Each object has its checksum validated and is stored in `/ostree/repo/objects/`.
Once the pull is complete, we have all the objects locally
we need to perform a deployment.
## Upgrades via external tools (e.g. package managers)
As mentioned in the introduction, OSTree is also designed to allow a
model where filesystem trees are computed on the client. It is
completely agnostic as to how those trees are generated; hey could be
computed with traditional packages, packages with post-deployment
scripts on top, or built by developers directly from revision control
locally, etc.
At a practical level, most package managers today (`dpkg` and `rpm`)
operate "live" on the currently booted filesystem. The way they could
work with OSTree is instead to take the list of installed packages in
the currently booted tree, and compute a new filesystem from that. A
later chapter describes in more details how this could work:
[adapting-existing.md](Adapting Existing Systems).
For the purposes of this section, let's assume that we have a
newly generated filesystem tree stored in the repo (which shares
storage with the existing booted tree). We can then move on to
checking it back out of the repo into a deployment.
## Assembling a new deployment directory
Given a commit to deploy, OSTree first allocates a directory for
it. This is of the form `/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`.
he $serial is normally 0, but if a
given commit is deployed more than once, it will be incremented.
his is supported because the previous deployment may have
configuration in `/etc`
hat we do not want to use or overwrite.
Now that we have a deployment directory, a 3-way merge is
performed between the (by default) currently booted deployment's
`/etc`, its default
configuration, and the new deployment (based on its `/usr/etc`).
## Atomically swapping boot configuration
At this point, a new deployment directory has been created as a
hardlink farm; the running system is untouched, and the bootloader
configuration is untouched. We want to add this deployment o the
"deployment list".
To support a more general case, OSTree supports atomic ransitioning
between arbitrary sets of deployments, with the restriction that the
currently booted deployment must always be in the new set. In the
normal case, we have exactly one deployment, which is the booted one,
and we want to add the new deployment to the list. A more complex
command might allow creating 100 deployments as part of one atomic
transaction, so that one can set up an automated system to bisect
across them.
## The bootversion
OSTree allows swapping between boot configurations by implementing the
"swapped directory pattern" in `/boot`. This means it is a symbolic
link to one of two directories `/ostree/boot.[0|1]`. To swap the
contents atomically, if the current version is `0`, we create
`/ostree/boot.1`, populate it with the new contents, then atomically
swap the symbolic link. Finally, the old contents can be garbage
collected at any point.
## The /ostree/boot directory
However, we want to optimize for the case where we the set of
kernel/initramfs pairs is the same between both the old and new
deployment lists. This happens when doing an upgrade that does not
include the kernel; think of a simple translation update. OSTree
optimizes for this case because on some systems `/boot` may be on a
separate medium such as flash storage not optimized for significant
amounts of write traffic.
To implement this, OSTree also maintains the directory
`/ostree/boot.<replaceable>bootversion</replaceable>`, which is a set
of symbolic links to the deployment directories. The
<replaceable>bootversion</replaceable> here must match the version of
`/boot`. However, in order to allow atomic transitions of
<emphasis>this</emphasis> directory, this is also a swapped directory,
so just like `/boot`, it has a version of `0` or `1` appended.
Each bootloader entry has a special `ostree=` argument which refers to
one of these symbolic links. This is parsed at runtime in the
initramfs.
docs/manual/deployment.md
+90
View File
@@ -0,0 +1,90 @@
# Deployments
## Overview
Built on top of the OSTree versioning filesystem core is a layer
that knows how to deploy, parallel install, and manage Unix-like
operating systems (accessible via `ostree admin`). The core content of these operating systems
are treated as read-only, but they transparently share storage.
A deployment is physically located at a path of the form
`/ostree/deploy/$osname/deploy/$checksum`.
OSTree is designed to boot directly into exactly one deployment
at a time; each deployment is intended to be a target for
`chroot()` or equivalent.
### "osname": Group of deployments that share /var</title>
Each deployment is grouped in exactly one "osname". From above, you
can see that an osname is physically represented in the
`/ostree/deploy/$osname` directory. For example, OSTree can allow
parallel installing Debian in `/ostree/deploy/debian` and Red Hat
Enterprise Linux in `/ostree/deploy/rhel` (subject to operating system
support, present released versions of these operating systems may not
support this).
Each osname has exactly one copy of the traditional Unix `/var`,
stored physically in `/ostree/deploy/$osname/var`. OSTree provides
support tools for `systemd` to create a Linux bind mount that ensures
the booted deployment sees the shared copy of `/var`.
OSTree does not touch the contents of `/var`. Operating system
components such as daemon services are required to create any
directories they require there at runtime
(e.g. `/var/cache/$daemonname`), and to manage upgrading data formats
inside those directories.
### Contents of a deployment
A deployment begins with a specific commit (represented as a
SHA256 hash) in the OSTree repository in `/ostree/repo`. This commit refers
to a filesystem tree that represents the underlying basis of a
deployment. For short, we will call this the "tree", to
distinguish it from the concept of a deployment.
First, the tree must include a kernel stored as
`/boot/vmlinuz-$checksum`. The checksum should be a SHA256 hash of
the kernel contents; it must be pre-computed before storing the kernel
in the repository. Optionally, the tree can contain an initramfs,
stored as `/boot/initramfs-$checksum`. If this exists, the checksum
must include both the kernel and initramfs contents. OSTree will use
this to determine which kernels are shared. The rationale for this is
to avoid computing checksums on the client by default.
The deployment should not have a traditional UNIX `/etc`; instead, it
should include `/usr/etc`. This is the "default configuration". When
OSTree creates a deployment, it performs a 3-way merge using the
<emphasis>old</emphasis> default configuration, the active system's
`/etc`, and the new default configuration. In the final filesystem
tree for a deployment then, `/etc` is a regular writable directory.
Besides the exceptions of `/var` and `/etc` then, the rest of the
contents of the tree are checked out as hard links into the
repository. It's strongly recommended that operating systems ship all
of their content in `/usr`, but this is not a hard requirement.
Finally, a deployment may have a `.origin` file, stored next to its
directory. This file tells `ostree admin upgrade` how to upgrade it.
At the moment, OSTree only supports upgrading a single refspec.
However, in the future OSTree may support a syntax for composing
layers of trees, for example.
### The system /boot
While OSTree parallel installs deployments cleanly inside the
`/ostree` directory, ultimately it has to control the system's `/boot`
directory. The way this works is via the
[Boot Loader Specification](http://www.freedesktop.org/wiki/Specifications/BootLoaderSpec),
which is a standard for bootloader-independent drop-in configuration
files.
When a tree is deployed, it will have a configuration file generated
of the form
`/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. This
configuration file will include a special `ostree=` kernel argument
that allows the initramfs to find (and `chroot()` into) the specified
deployment.
At present, not all bootloaders implement the BootLoaderSpec, so
OSTree contains code for some of these to regenerate native config
files (such as `/boot/syslinux/syslinux.conf` based on the entries.
docs/manual/formats.md
+181
View File
@@ -0,0 +1,181 @@
# OSTree data formats
## On the topic of "smart servers"
One really crucial difference between OSTree and git is that git has a
"smart server". Even when fetching over `https://`, it isn't just a
static webserver, but one that e.g. dynamically computes and
compresses pack files for each client.
In contrast, the author of OSTree feels that for operating system
updates, many deployments will want to use simple static webservers,
the same target most package systems were designed to use. The
primary advantages are security and compute efficiency. Services like
Amazon S3 and CDNs are a canonical target, as well as a stock static
nginx server.
## The archive-z2 format
In the [repo](repo) section, the concept of objects was introduced,
where file/content objects are checksummed and managed individually.
(Unlike a package system, which operates on compressed aggregates).
The archive-z2 format simply gzip-compresses each content object.
Metadata objects are stored uncompressed. This means that it's easy
to serve via static HTTP.
When you commit new content, you will see new `.filez` files appearing
in `objects/`.
## archive-z2 efficiency
The advantages of `archive-z2`:
- It's easy to understand and implement
- Can be served directly over plain HTTP by a static webserver
- Clients can download/unpack updates incrementally
- Space efficient on the server
The biggest disadvantage of this format is that for a client to
perform an update, one HTTP request per changed file is required. In
some scenarios, this actually isn't bad at all, particularly with
techniques to reduce HTTP overhead, such as
[HTTP/2](https://en.wikipedia.org/wiki/HTTP/2).
In order to make this format work well, you should design your content
such that large data that changes infrequently (e.g. graphic images)
are stored separately from small frequently changing data (application
code).
Other disadvantages of `archive-z2`:
- It's quite bad when clients are performing an initial pull (without HTTP/2),
- One doesn't know the total size (compressed or uncompressed) of content
before downloading everything
## Aside: the bare and bare-user formats
The most common operation is to pull from an `archive-z2` repository
into a `bare` or `bare-user` formatted repository. These latter two
are not compressed on disk. In other words, pulling to them is
similar to unpacking (but not installing) an RPM/deb package.
The `bare-user` format is a bit special in that the uid/gid and xattrs
from the content are ignored. This is primarily useful if you want to
have the same OSTree-managed content that can be run on a host system
or an unprivileged container.
## Static deltas
OSTree itself was originally focused on a continous delivery model, where
client systems are expected to update regularly. However, many OS vendors
would like to supply content that's updated e.g. once a month or less often.
For this model, we can do a lot better to support batched updates than
a basic `archive-z2` repo. However, we still want to preserve the
model of "static webserver only". Given this, OSTree has gained the
concept of a "static delta".
These deltas are targeted to be a delta between two specific commit
objects, including "bsdiff" and "rsync-style" deltas within a content
object. Static deltas also support `from NULL`, where the client can
more efficiently download a commit object from scratch.
Effectively, we're spending server-side storage (and one-time compute
cost), and gaining efficiency in client network bandwith.
## Static delta repository layout
Since static deltas may not exist, the client first needs to attempt
to locate one. Suppose a client wants to retrieve commit `${new}`
while currently running `${current}`.
The first thing to understand is that in order to save space, these
two commits are "modified base64" - the `/` character is replaced with
`_`.
Like the commit objects, a "prefix directory" is used to make
management easier for filesystem tools
A delta is named `$(mbase64 $from)-$(mbase64 $to)`, for example
`GpTyZaVut2jXFPWnO4LJiKEdRTvOw_mFUCtIKW1NIX0-L8f+VVDkEBKNc1Ncd+mDUrSVR4EyybQGCkuKtkDnTwk`,
which in sha256 format is
`1a94f265a56eb768d714f5a73b82c988a11d453bcec3f985502b48296d4d217d-2fc7fe5550e410128d73535c77e98352b495478132c9b4060a4b8ab640e74f09`.
Finally, the actual content can be found in
`deltas/$fromprefix/$fromsuffix-$to`.
## Static delta internal structure
A delta is itself a directory. Inside, there is a file called
`superblock` which contains metadata. The rest of the files will be
integers bearing packs of content.
The file format of static deltas should be currently considered an
OSTree implementation detail. Obviously, nothing stops one from
writing code which is compatible with OSTree today. However, we would
like the flexibility to expand and change things, and having multiple
codebases makes that more problematic. Please contact the authors
with any requests.
That said, one critical thing to understand about the design is that
delta payloads are a bit more like "restricted programs" than they are
raw data. There's a "compilation" phase which generates output that
the client executes.
This "updates as code" model allows for multiple content generation
strategies. The design of this was inspired by that of Chromium:
[http://dev.chromium.org/chromium-os/chromiumos-design-docs/filesystem-autoupdate](ChromiumOS
autoupdate).
### The delta superblock
The superblock contains:
- arbitrary metadata
- delta generation timestamp
- the new commit object
- An array of recursive deltas to apply
- An array of per-part metadata, including total object sizes (compressed and uncompressed),
- An array of fallback objects
Let's define a delta part, then return to discuss details:
## A delta part
A delta part is a combination of a raw blob of data, plus a very
restricted bytecode that operates on it. Say for example two files
happen to share a common section. It's possible for the delta
compilation to include that section once in the delta data blob, then
generate instructions to write out that blob twice when generating
both objects.
Realistically though, it's very common for most of a delta to just be
"stream of new objects" - if one considers it, it doesn't make sense
to have too much duplication inside operating system content at this
level.
So then, what's more interesting is that OSTree static deltas support
a per-file delta algorithm called
[bsdiff](https://github.com/mendsley/bsdiff) that most notably works
well on executable code.
The current delta compiler scans for files with maching basenamesin
each commit that have a similar size, and attempts a bsdiff between
them. (It would make sense later to have a build system provide a
hint for this - for example, files within a same package).
A generated bsdiff is included in the payload blob, and applying it is
an instruction.
## Fallback objects
It's possible for there to be large-ish files which might be resistant
to bsdiff. A good example is that it's common for operating systems
to use an "initramfs", which is itself a compressed filesystem. This
"internal compression" defeats bsdiff analysis.
For these types of objects, the delta superblock contains an array of
"fallback objects". These objects aren't included in the delta
parts - the client simply fetches them from the underlying `.filez`
object.
docs/manual/introduction.md
+110
View File
@@ -0,0 +1,110 @@
# OSTree Overview
## Introduction
OSTree an upgrade system for Linux-based operating systems that
performs atomic upgrades of complete filesystem trees. It is
not a package system; rather, it is intended to complement them.
A primary model is composing packages on a server, and then
replicating them to clients.
The underlying architecture might be summarized as "git for
operating system binaries". It operates in userspace, and will
work on top of any Linux filesystem. At its core is a git-like
content-addressed object store, and layered on top of that is
bootloader configuration, management of
`/etc`, and other functions to perform an
upgrade beyond just replicating files.
You can use OSTree standalone in the pure replication model,
but another approach is to add a package manager on top,
thus creating a hybrid tree/package system.
## Comparison with "package managers"
Because OSTree is designed for deploying core operating
systems, a comparison with traditional "package managers" such
as dpkg and rpm is illustrative. Packages are traditionally
composed of partial filesystem trees with metadata and scripts
attached, and these are dynamically assembled on the client
machine, after a process of dependency resolution.
In contrast, OSTree only supports recording and deploying
*complete* (bootable) filesystem trees. It
has no built-in knowledge of how a given filesystem tree was
generated or the origin of individual files, or dependencies,
descriptions of individual components. Put another way, OSTree
only handles delivery and deployment; you will likely still want
to include inside each tree metadata about the individual
components that went into the tree. For example, a system
administrator may want to know what version of OpenSSL was
included in your tree, so you should support the equivalent of
`rpm -q` or `dpkg -L`.
The OSTree core emphasizes replicating read-only OS trees via
HTTP, and where the OS includes (if desired) an entirely
separate mechanism to install applications, stored in `/var` if they're system global, or
`/home` for per-user
application installation. An example application mechanism is
http://docker.io/
However, it is entirely possible to use OSTree underneath a
package system, where the contents of `/usr` are computed on the client.
For example, when installing a package, rather than changing the
currently running filesystem, the package manager could assemble
a new filesystem tree that layers the new packages on top of a
base tree, record it in the local OSTree repository, and then
set it up for the next boot. To support this model, OSTree
provides an (introspectable) C shared library.
## Comparison with block/image replication
OSTree shares some similarity with "dumb" replication and
stateless deployments, such as the model common in "cloud"
deployments where nodes are booted from an (effectively)
readonly disk, and user data is kept on a different volumes.
The advantage of "dumb" replication, shared by both OSTree and
the cloud model, is that it's *reliable*
and *predictable*.
But unlike many default image-based deployments, OSTree supports
exactly two persistent writable directories that are preserved across
upgrades: `/etc` and `/var`.
Because OSTree operates at the Unix filesystem layer, it works
on top of any filesystem or block storage layout; it's possible
to replicate a given filesystem tree from an OSTree repository
into plain ext4, BTRFS, XFS, or in general any Unix-compatible
filesystem that supports hard links. Note: OSTree will
transparently take advantage of some BTRFS features if deployed
on it.
## Atomic transitions between parallel-installable read-only filesystem trees
Another deeply fundamental difference between both package
managers and image-based replication is that OSTree is
designed to parallel-install *multiple versions* of multiple
*independent* operating systems. OSTree
relies on a new toplevel `ostree` directory; it can in fact
parallel install inside an existing OS or distribution
occupying the physical `/` root.
On each client machine, there is an OSTree repository stored
in `/ostree/repo`, and a set of "deployments" stored in `/ostree/deploy/$OSNAME/$CHECKSUM`.
Each deployment is primarily composed of a set of hardlinks
into the repository. This means each version is deduplicated;
an upgrade process only costs disk space proportional to the
new files, plus some constant overhead.
The model OSTree emphasizes is that the OS read-only content
is kept in the classic Unix `/usr`; it comes with code to
create a Linux read-only bind mount to prevent inadvertent
corruption. There is exactly one `/var` writable directory shared
between each deployment for a given OS. The OSTree core code
does not touch content in this directory; it is up to the code
in each operating system for how to manage and upgrade state.
Finally, each deployment has its own writable copy of the
configuration store `/etc`. On upgrade, OSTree will
perform a basic 3-way diff, and apply any local changes to the
new copy, while leaving the old untouched.
docs/manual/repo.md
+90
View File
@@ -0,0 +1,90 @@
# Anatomy of an OSTree repository
## Core object types and data model
OSTree is deeply inspired by git; the core layer is a userspace
content-addressed versioning filesystem. It is worth taking some time
to familiarize yourself with
[Git Internals](http://git-scm.com/book/en/Git-Internals), as this
section will assume some knowledge of how git works.
Its object types are similar to git; it has commit objects and content
objects. Git has "tree" objects, whereas OSTree splits them into
"dirtree" and "dirmeta" objects. But unlike git, OSTree's checksums
are SHA256. And most crucially, its content objects include uid, gid,
and extended attributes (but still no timestamps).
### Commit objects
A commit object contains metadata such as a timestamp, a log
message, and most importantly, a reference to a
dirtree/dirmeta pair of checksums which describe the root
directory of the filesystem.
Also like git, each commit in OSTree can have a parent. It is
designed to store a history of your binary builds, just like git
stores a history of source control. However, OSTree also makes
it easy to delete data, under the assumption that you can
regenerate it from source code.
### Dirtree objects
A dirtree contains a sorted array of (filename, checksum)
pairs for content objects, and a second sorted array of
(filename, dirtree checksum, dirmeta checksum), which are
subdirectories.
### Dirmeta objects
In git, tree objects contain the metadata such as permissions
for their children. But OSTree splits this into a separate
object to avoid duplicating extended attribute listings.
### Content objects
Unlike the first three object types which are metadata, designed to be
`mmap()`ed, the content object has a separate internal header and
payload sections. The header contains uid, gid, mode, and symbolic
link target (for symlinks), as well as extended attributes. After the
header, for regular files, the content follows.
# Repository types and locations
Also unlike git, an OSTree repository can be in one of three separate
modes: `bare`, `bare-user`, and `archive-z2`. A bare repository is
one where content files are just stored as regular files; it's
designed to be the source of a "hardlink farm", where each operating
system checkout is merely links into it. If you want to store files
owned by e.g. root in this mode, you must run OSTree as root.
The `bare-user` is a later addition that is like `bare` in that files
are unpacked, but it can (and should generally) be created as
non-root. In this mode, extended metadata such as owner uid, gid, and
extended attributes are stored but not actually applied.
The `bare-user` mode is useful for build systems that run as non-root
but want to generate root-owned content, as well as non-root container
systems.
In contrast, the `archive-z2` mode is designed for serving via plain
HTTP. Like tar files, it can be read/written by non-root users.
On an OSTree-deployed system, the "system repository" is
`/ostree/repo`. It can be read by any uid, but only written by root.
Unless the `--repo` argument is given to the <command>ostree</command>
command, it will operate on the system repository.
## Refs
Like git, OSTree uses "refs" to which are text files that point to
particular commits (i.e. filesystem trees). For example, the
gnome-ostree operating system creates trees named like
`exampleos/buildmaster/x86_64-runtime` and
`exampleos/buildmaster/x86_64-devel-debug`. These two refs point to
two different generated filesystem trees. In this example, the
"runtime" tree contains just enough to run a basic system, and
"devel-debug" contains all of the developer tools and debuginfo.
The `ostree` supports a simple syntax using the carat `^` to refer to
the parent of a given commit. For example,
`exampleos/buildmaster/x86_64-runtime^` refers to the previous build,
and `exampleos/buildmaster/x86_64-runtime^^` refers to the one before
that.
libglnx/Makefile-libglnx.am
+1
View File
@@ -18,6 +18,7 @@
EXTRA_DIST += $(libglnx_srcpath)/README.md $(libglnx_srcpath)/COPYING
libglnx_la_SOURCES = \
$(libglnx_srcpath)/glnx-alloca.h \
$(libglnx_srcpath)/glnx-backport-autocleanups.h \
$(libglnx_srcpath)/glnx-backport-autoptr.h \
$(libglnx_srcpath)/glnx-backports.h \
libglnx/glnx-alloca.h
+47
View File
@@ -0,0 +1,47 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2014,2015 Colin Walters <walters@verbum.org>.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#pragma once
#include <stdlib.h>
#include <glib.h>
G_BEGIN_DECLS
/* Taken from https://github.com/systemd/systemd/src/basic/string-util.h
* at revision v228-666-gcf6c8c4
*/
#define glnx_strjoina(a, ...) \
({ \
const char *_appendees_[] = { a, __VA_ARGS__ }; \
char *_d_, *_p_; \
int _len_ = 0; \
unsigned _i_; \
for (_i_ = 0; _i_ < G_N_ELEMENTS(_appendees_) && _appendees_[_i_]; _i_++) \
_len_ += strlen(_appendees_[_i_]); \
_p_ = _d_ = alloca(_len_ + 1); \
for (_i_ = 0; _i_ < G_N_ELEMENTS(_appendees_) && _appendees_[_i_]; _i_++) \
_p_ = stpcpy(_p_, _appendees_[_i_]); \
*_p_ = 0; \
_d_; \
})
G_END_DECLS
libglnx/glnx-backport-autocleanups.h
+15 -8
View File
@@ -31,13 +31,6 @@ g_autoptr_cleanup_generic_gfree (void *p)
g_free (*pp);
}
static inline void
g_autoptr_cleanup_gstring_free (GString *string)
{
if (string)
g_string_free (string, TRUE);
}
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GAsyncQueue, g_async_queue_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GBookmarkFile, g_bookmark_file_free)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GBytes, g_bytes_unref)
@@ -55,7 +48,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(GPtrArray, g_ptr_array_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMainContext, g_main_context_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMainLoop, g_main_loop_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GSource, g_source_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GString, g_autoptr_cleanup_gstring_free)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMappedFile, g_mapped_file_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMarkupParseContext, g_markup_parse_context_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(gchar, g_free)
@@ -113,5 +105,20 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(GTlsDatabase, g_object_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GTlsInteraction, g_object_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GDBusConnection, g_object_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GDBusMessage, g_object_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GZlibCompressor, g_object_unref)
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GZlibDecompressor, g_object_unref)
#endif
#if !GLIB_CHECK_VERSION(2, 45, 8)
static inline void
g_autoptr_cleanup_gstring_free (GString *string)
{
if (string)
g_string_free (string, TRUE);
}
G_DEFINE_AUTOPTR_CLEANUP_FUNC(GString, g_autoptr_cleanup_gstring_free)
#endif
libglnx/glnx-console.c
+2 -2
View File
@@ -241,7 +241,7 @@ glnx_console_progress_text_percent (const char *text,
if (textlen > 0)
{
fwrite (text, 1, textlen - 1, stdout);
fwrite (text, 1, textlen, stdout);
fputc (' ', stdout);
}
@@ -285,5 +285,5 @@ glnx_console_unlock (GLnxConsoleRef *console)
if (console->is_tty)
fputc ('\n', stdout);
locked = FALSE;
locked = console->locked = FALSE;
}
libglnx/glnx-console.h
+2 -1
View File
@@ -45,7 +45,8 @@ guint glnx_console_columns (void);
static inline void
glnx_console_ref_cleanup (GLnxConsoleRef *p)
{
glnx_console_unlock (p);
if (p->locked)
glnx_console_unlock (p);
}
G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(GLnxConsoleRef, glnx_console_ref_cleanup)
libglnx/glnx-dirfd.c
+49
View File
@@ -168,6 +168,8 @@ glnx_dirfd_iterator_next_dent (GLnxDirFdIterator *dfd_iter,
gboolean ret = FALSE;
GLnxRealDirfdIterator *real_dfd_iter = (GLnxRealDirfdIterator*) dfd_iter;
g_return_val_if_fail (out_dent, FALSE);
if (g_cancellable_set_error_if_cancelled (cancellable, error))
goto out;
@@ -189,6 +191,53 @@ glnx_dirfd_iterator_next_dent (GLnxDirFdIterator *dfd_iter,
return ret;
}
/**
* glnx_dirfd_iterator_next_dent_ensure_dtype:
* @dfd_iter: A directory iterator
* @out_dent: (out) (transfer none): Pointer to dirent; do not free
* @cancellable: Cancellable
* @error: Error
*
* A variant of @glnx_dirfd_iterator_next_dent, which will ensure the
* `dent->d_type` member is filled in by calling `fstatat`
* automatically if the underlying filesystem type sets `DT_UNKNOWN`.
*/
gboolean
glnx_dirfd_iterator_next_dent_ensure_dtype (GLnxDirFdIterator *dfd_iter,
struct dirent **out_dent,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
struct dirent *ret_dent;
g_return_val_if_fail (out_dent, FALSE);
if (!glnx_dirfd_iterator_next_dent (dfd_iter, out_dent, cancellable, error))
goto out;
ret_dent = *out_dent;
if (ret_dent)
{
if (ret_dent->d_type == DT_UNKNOWN)
{
struct stat stbuf;
if (TEMP_FAILURE_RETRY (fstatat (dfd_iter->fd, ret_dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW)) != 0)
{
glnx_set_error_from_errno (error);
goto out;
}
ret_dent->d_type = IFTODT (stbuf.st_mode);
}
}
ret = TRUE;
out:
return ret;
}
/**
* glnx_dirfd_iterator_clear:
* @dfd_iter: Iterator, will be de-initialized
libglnx/glnx-dirfd.h
+4
View File
@@ -60,6 +60,10 @@ gboolean glnx_dirfd_iterator_next_dent (GLnxDirFdIterator *dfd_iter,
struct dirent **out_dent,
GCancellable *cancellable,
GError **error);
gboolean glnx_dirfd_iterator_next_dent_ensure_dtype (GLnxDirFdIterator *dfd_iter,
struct dirent **out_dent,
GCancellable *cancellable,
GError **error);
void glnx_dirfd_iterator_clear (GLnxDirFdIterator *dfd_iter);
G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(GLnxDirFdIterator, glnx_dirfd_iterator_clear)
libglnx/glnx-fdio.c
+3 -3
View File
@@ -354,7 +354,7 @@ static int btrfs_reflink(int infd, int outfd) {
return 0;
}
static int loop_write(int fd, const void *buf, size_t nbytes) {
int glnx_loop_write(int fd, const void *buf, size_t nbytes) {
const uint8_t *p = buf;
g_return_val_if_fail(fd >= 0, -1);
@@ -437,7 +437,7 @@ static int copy_bytes(int fdf, int fdt, off_t max_bytes, bool try_reflink) {
if (n == 0) /* EOF */
break;
r = loop_write(fdt, buf, (size_t) n);
r = glnx_loop_write(fdt, buf, (size_t) n);
if (r < 0)
return r;
}
@@ -685,7 +685,7 @@ glnx_file_replace_contents_with_perms_at (int dfd,
goto out;
}
if ((r = loop_write (fd, buf, len)) != 0)
if ((r = glnx_loop_write (fd, buf, len)) != 0)
{
errno = -r;
glnx_set_error_from_errno (error);
libglnx/glnx-fdio.h
+3
View File
@@ -102,6 +102,9 @@ glnx_readlinkat_malloc (int dfd,
GCancellable *cancellable,
GError **error);
int
glnx_loop_write (int fd, const void *buf, size_t nbytes);
typedef enum {
GLNX_FILE_COPY_OVERWRITE,
GLNX_FILE_COPY_NOXATTRS,
libglnx/libglnx.h
+1
View File
@@ -24,6 +24,7 @@
G_BEGIN_DECLS
#include <glnx-alloca.h>
#include <glnx-local-alloc.h>
#include <glnx-backport-autocleanups.h>
#include <glnx-backports.h>
doc/ostree-admin-cleanup.xml → man/ostree-admin-cleanup.xml
View File
File renamed without changes.
doc/ostree-admin-config-diff.xml → man/ostree-admin-config-diff.xml
View File
File renamed without changes.
doc/ostree-admin-deploy.xml → man/ostree-admin-deploy.xml
View File
File renamed without changes.
doc/ostree-admin-init-fs.xml → man/ostree-admin-init-fs.xml
View File
File renamed without changes.
doc/ostree-admin-instutil.xml → man/ostree-admin-instutil.xml
View File
File renamed without changes.
doc/ostree-admin-os-init.xml → man/ostree-admin-os-init.xml
View File
File renamed without changes.
doc/ostree-admin-set-origin.xml → man/ostree-admin-set-origin.xml
View File
File renamed without changes.
doc/ostree-admin-status.xml → man/ostree-admin-status.xml
View File
File renamed without changes.
doc/ostree-admin-switch.xml → man/ostree-admin-switch.xml
View File
File renamed without changes.
doc/ostree-admin-undeploy.xml → man/ostree-admin-undeploy.xml
View File
File renamed without changes.
doc/ostree-admin-upgrade.xml → man/ostree-admin-upgrade.xml
View File
File renamed without changes.
doc/ostree-admin.xml → man/ostree-admin.xml
View File
File renamed without changes.
doc/ostree-cat.xml → man/ostree-cat.xml
View File
File renamed without changes.
doc/ostree-checkout.xml → man/ostree-checkout.xml
View File
File renamed without changes.
doc/ostree-checksum.xml → man/ostree-checksum.xml
View File
File renamed without changes.
doc/ostree-commit.xml → man/ostree-commit.xml
View File
File renamed without changes.
doc/ostree-config.xml → man/ostree-config.xml
View File
File renamed without changes.
doc/ostree-diff.xml → man/ostree-diff.xml
View File
File renamed without changes.
man/ostree-export.xml
+70
View File
@@ -0,0 +1,70 @@
<?xml version='1.0'?> <!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
Copyright 2016 Colin Walters <walters@verbum.org>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the
Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.
-->
<refentry id="ostree">
<refentryinfo>
<title>ostree export</title>
<productname>OSTree</productname>
<authorgroup>
<author>
<contrib>Developer</contrib>
<firstname>Colin</firstname>
<surname>Walters</surname>
<email>walters@verbum.org</email>
</author>
</authorgroup>
</refentryinfo>
<refmeta>
<refentrytitle>ostree export</refentrytitle>
<manvolnum>1</manvolnum>
</refmeta>
<refnamediv>
<refname>ostree-export</refname>
<refpurpose>Generate a tar archive from an OSTree commit</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>ostree export</command> <arg choice="opt" rep="repeat">OPTIONS</arg> <arg choice="req">BRANCH</arg>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
This command generates a GNU tar formatted archive from an
OSTree commit. This is useful for cases like backups,
converting OSTree commits into Docker images, and the like.
</para>
</refsect1>
<refsect1>
<title>Example</title>
<para><command>$ ostree export exampleos/x86_64/standard | gzip > exampleos-standard.tar.gz</command></para>
</refsect1>
</refentry>
doc/ostree-fsck.xml → man/ostree-fsck.xml
View File
File renamed without changes.
doc/ostree-gpg-sign.xml → man/ostree-gpg-sign.xml
View File
File renamed without changes.
doc/ostree-init.xml → man/ostree-init.xml
View File
File renamed without changes.
doc/ostree-log.xml → man/ostree-log.xml
View File
File renamed without changes.
doc/ostree-ls.xml → man/ostree-ls.xml
View File
File renamed without changes.
doc/ostree-prune.xml → man/ostree-prune.xml
View File
File renamed without changes.
doc/ostree-pull-local.xml → man/ostree-pull-local.xml
View File
File renamed without changes.
doc/ostree-pull.xml → man/ostree-pull.xml
+25 -2
View File
@@ -111,14 +111,37 @@ Boston, MA 02111-1307, USA.
<title>Description</title>
<para>
Downloads all content corresponding to the provided branch or commit from the given remote.
This command can retrieve just a specific commit, or go all
the way to performing a full mirror of the remote
repository. If no <literal>BRANCH</literal> is specified,
all branches are retrieved.
</para>
<para>
A special syntax in the <literal>@</literal> character allows
specifying a specific commit to retrieve from a branch. This
</para>
</refsect1>
<refsect1>
<title>Example</title>
<para><command>$ ostree pull remote_name</command></para>
<para><command>$ ostree --repo=repo pull --depth=-1 --mirror remote_name</command></para>
<para>Perform a complete mirror of the remote. (This is
likely most useful if your repository is also
<literal>archive-z2</literal> mode)</para>
<para><command>$ ostree --repo=repo pull remote_name exampleos/x86_64/standard</command></para>
<para>Fetch the most recent commit to <literal>exampleos/x86_64/standard</literal>.</para>
<para><command>$ ostree --repo=repo pull remote_name exampleos/x86_64/standard@98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4</command></para>
<para>Download the specific commit starting with
<literal>98ea6e</literal> as if it was the latest commit for
<literal>exampleos/x86_64/standard</literal>.</para>
</refsect1>
</refentry>
doc/ostree-refs.xml → man/ostree-refs.xml
View File
File renamed without changes.
doc/ostree-remote.xml → man/ostree-remote.xml
View File
File renamed without changes.
doc/ostree-reset.xml → man/ostree-reset.xml
View File
File renamed without changes.
doc/ostree-rev-parse.xml → man/ostree-rev-parse.xml
View File
File renamed without changes.
doc/ostree-show.xml → man/ostree-show.xml
View File
File renamed without changes.
doc/ostree-static-delta.xml → man/ostree-static-delta.xml
View File
File renamed without changes.
doc/ostree-summary.xml → man/ostree-summary.xml
View File
File renamed without changes.
doc/ostree-trivial-httpd.xml → man/ostree-trivial-httpd.xml
View File
File renamed without changes.
doc/ostree.repo-config.xml → man/ostree.repo-config.xml
View File
File renamed without changes.
doc/ostree.repo.xml → man/ostree.repo.xml
View File
File renamed without changes.
doc/ostree.xml → man/ostree.xml
View File
File renamed without changes.
man/rofiles-fuse.xml
+104
View File
@@ -0,0 +1,104 @@
<?xml version='1.0'?> <!--*-nxml-*-->
<!DOCTYPE refentry PUBLIC "-//OASIS//DTD DocBook XML V4.2//EN"
"http://www.oasis-open.org/docbook/xml/4.2/docbookx.dtd">
<!--
Copyright 2016 Colin Walters <walters@verbum.org>
This library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2 of the License, or (at your option) any later version.
This library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with this library; if not, write to the
Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.
-->
<refentry id="ostree">
<refentryinfo>
<title>rofiles-fuse</title>
<productname>rofiles-fuse</productname>
<authorgroup>
<author>
<contrib>Developer</contrib>
<firstname>Colin</firstname>
<surname>Walters</surname>
<email>walters@verbum.org</email>
</author>
</authorgroup>
</refentryinfo>
<refmeta>
<refentrytitle>rofiles-fuse</refentrytitle>
<manvolnum>1</manvolnum>
</refmeta>
<refnamediv>
<refname>rofiles-fuse</refname>
<refpurpose>Use FUSE to create a view where directories are writable, files are immutable</refpurpose>
</refnamediv>
<refsynopsisdiv>
<cmdsynopsis>
<command>rofiles-fuse SRCDIR MNTPOINT</command>
</cmdsynopsis>
</refsynopsisdiv>
<refsect1>
<title>Description</title>
<para>
Creating a checkout from an OSTree repository by default
uses hard links, which means an in-place mutation to any
file corrupts the repository and all checkouts. This can be
problematic if one wishes to run arbitrary programs against
such a checkout. For example, RPM <literal>%post</literal>
scripts or equivalent.
</para>
<para>
In the case where one wants to create a tree commit derived
from other content, using <command>rofiles-fuse</command> in
concert with <command>ostree commit
--link-checkout-speedup</command> (or the underlying API)
can ensure that only new files are checksummed.
</para>
</refsect1>
<refsect1>
<title>Example: Update an OSTree commit</title>
<programlisting>
# Initialize a checkout and mount
$ ostree --repo=repo checkout somebranch branch-checkout
$ mkdir mnt
$ rofiles-fuse branch-checkout mnt
# Now, arbitrary changes to mnt/ are reflected in branch-checkout
$ echo somenewcontent > mnt/anewfile
$ mkdir mnt/anewdir
$ rm mnt/someoriginalcontent -rf
# Commit and cleanup
$ fusermount -u mnt
$ ostree --repo=repo commit --link-checkout-speedup -b somebranch -s 'Commit new content' --tree=dir=branch-checkout
$ rm mnt branch-checkout -rf
</programlisting>
</refsect1>
<refsect1>
<title>See Also</title>
<para>
<citerefentry><refentrytitle>ostree</refentrytitle><manvolnum>1</manvolnum></citerefentry>
</para>
</refsect1>
</refentry>
manual-tests/static-delta-generate-crosscheck.sh
Executable
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env bash
#
# Generate multiple variants of static deltas in a repo, then use them
# to upgrade in a test repository, verifying that fsck works.
#
# This test is manual so it's easy to run against arbitrary content.
set -euo pipefail
repo=$(pwd)/${1}
shift
branch=${1}
shift
from=$(ostree --repo=${repo} rev-parse "${branch}"^)
to=$(ostree --repo=${repo} rev-parse ${branch})
tmpdir=$(mktemp -d /var/tmp/ostree-delta-check.XXXXXX)
cd ${tmpdir}
touch ${tmpdir}/.tmp
echo "Using tmpdir ${tmpdir}"
cleanup_tmpdir() {
if test -f ${tmpdir}/.tmp; then
rm -rf ${tmpdir}
fi
}
if test -z "${PRESERVE_TMP:-}"; then
trap cleanup_tmpdir EXIT
fi
fatal() {
echo "$@"
exit 1
}
assert_streq() {
if ! test $1 = $2; then
fatal "assertion failed: $1 = $2"
fi
}
validate_delta_options() {
mkdir testrepo
ostree --repo=testrepo init --mode=bare-user
ostree --repo=testrepo remote add --set=gpg-verify=false local file://${repo}
ostree --repo=${repo} static-delta generate $@ ${from} ${to}
ostree --repo=testrepo pull local ${branch}@${from}
assert_streq $(ostree --repo=testrepo rev-parse ${branch}) ${from}
ostree --repo=testrepo pull local ${branch}
assert_streq $(ostree --repo=testrepo rev-parse ${branch}) ${to}
ostree --repo=testrepo fsck
rm testrepo -rf
}
set -x
validate_delta_options
validate_delta_options --inline
validate_delta_options --disable-bsdiff
mkdocs.yml
+11
View File
@@ -0,0 +1,11 @@
site_name: My Docs
pages:
- Home: 'index.md'
- Contributing: 'CONTRIBUTING.md'
- Manual:
- Introduction: 'manual/introduction.md'
- Repository: 'manual/repo.md'
- Deployments: 'manual/deployment.md'
- Atomic Upgrades: 'manual/atomic-upgrades.md'
- Adapting Existing Systems: 'manual/adapting-existing.md'
- Formats: 'manual/formats.md'
packaging/ostree.spec.in
+14
View File
@@ -19,11 +19,14 @@ BuildRequires: pkgconfig(libgsystem)
BuildRequires: pkgconfig(e2p)
# Extras
BuildRequires: pkgconfig(libarchive)
BuildRequires: pkgconfig(fuse)
BuildRequires: pkgconfig(libselinux)
BuildRequires: libcap-devel
BuildRequires: gpgme-devel
BuildRequires: pkgconfig(systemd)
BuildRequires: /usr/bin/g-ir-scanner
BuildRequires: dracut
BuildRequires: bison
# Runtime requirements
Requires: dracut
@@ -53,6 +56,14 @@ Requires: grub2
%description grub2
GRUB2 integration for OSTree
%package fuse
Summary: FUSE utilities for OSTree
Group: Development/Libraries
Requires: fuse
%description fuse
%{summary}
%prep
%setup -q -n ostree-%{version}
@@ -106,3 +117,6 @@ rm -rf $RPM_BUILD_ROOT
%files grub2
%{_sysconfdir}/grub.d/*ostree
%{_libexecdir}/ostree/grub2*
%files fuse
%{_bindir}/rofiles-fuse
src/libostree/README-deltas.md
-158
View File
@@ -1,158 +0,0 @@
OSTree Static Object Deltas
===========================
Currently, OSTree's "archive-z2" mode stores both metadata and content
objects as individual files in the filesystem. Content objects are
zlib-compressed.
The advantage of this is model are:
0) It's easy to understand and implement
1) Can be served directly over plain HTTP by a static webserver
2) Space efficient on the server
However, it can be inefficient both for large updates and small ones:
0) For large tree changes (such as going from -runtime to
-devel-debug, or major version upgrades), this can mean thousands
and thousands of HTTP requests. The overhead for that is very
large (until SPDY/HTTP2.0), and will be catastrophically bad if the
webserver is not configured with KeepAlive.
1) Small changes (typo in gnome-shell .js file) still require around
5 metadata HTTP requests, plus a redownload of the whole file.
Why not smart servers?
======================
Smart servers (custom daemons, or just CGI scripts) as git has are not
under consideration for this proposal. OSTree is designed for the
same use case as GNU/Linux distribution package systems are, where
content is served by a network of volunteer mirrors that will
generally not run custom code.
In particular, Amazon S3 style dumb content servers is a very
important use case, as is being able to apply updates from static
media like DVD-ROM.
Finding Static Deltas
=====================
Since static deltas may not exist, the client first needs to attempt
to locate one. Suppose a client wants to retrieve commit ${new} while
currently running ${current}. The first thing to fetch is the delta
metadata, called "meta". It can be found at
${repo}/deltas/${current}-${new}/meta.
FIXME: GPG signatures (.metameta?) Or include commit object in meta?
But we would then be forced to verify the commit only after processing
the entirety of the delta, which is dangerous. I think we need to
require signing deltas.
Delta Bytecode Format
=====================
A delta-part has the following form:
byte compression-type (0 = none, 'g' = gzip')
REPEAT[(varint size, delta-part-content)]
delta-part-content:
byte[] payload
ARRAY[operation]
The rationale for having delta-part is that it allows easy incremental
resumption of downloads. The client can look at the delta descriptor
and skip downloading delta-parts for which it already has the
contained objects. This is better than simply resuming a gigantic
file because if the client decides to fetch a slightly newer version,
it's very probable that some of the downloading we've already done is
still useful.
For the actual delta payload, it comes as a stream of pair of
(payload, operation) so that it can be processed while being
decompressed.
Finally, the delta-part-content is effectively a high level bytecode
for a stack-oriented machine. It iterates on the array of objects in
order. The following operations are available:
FETCH
Fall back to fetching the current object individually. Move
to the next object.
WRITE(array[(varint offset, varint length)])
Write from current input target (default payload) to output.
GUNZIP(array[(varint offset, varint length)])
gunzip from current input target (default payload) to output.
CLOSE
Close the current output target, and proceed to the next; if the
output object was a temporary, the output resets to the current
object.
# Change the input source to an object
READOBJECT(csum object)
Set object as current input target
# Change the input source to payload
READPAYLOAD
Set payload as current input target
Compiling Deltas
================
After reading the above, you may be wondering how we actually *make*
these deltas. I envison a strategy similar to that employed by
Chromium autoupdate:
http://www.chromium.org/chromium-os/chromiumos-design-docs/autoupdate-details
Something like this would be a useful initial algorithm:
1) Compute the set of added objects NEW
2) For each object in NEW:
- Look for a the set of "superficially similar" objects in the
previous tree, using heuristics based first on filename (including
prefix), then on size. Call this set CANDIDATES.
For each entry in CANDIDATES:
- Try doing a bup/librsync style rolling checksum, and compute the
list of changed blocks.
- Try gzip-compressing it
3) Choose the lowest cost method for each NEW object, and partition
the program for each method into deltapart-sized chunks.
However, there are many other possibilities, that could be used in a
hybrid mode with the above. For example, we could try to find similar
objects, and gzip them together. This would be a *very* useful
strategy for things like the 9000 Boost headers which have massive
amounts of redundant data.
Notice too that the delta format supports falling back to retrieving
individual objects. For cases like the initramfs which is compressed
inside the tree with gzip, we're not going to find an efficient way to
sync it, so the delta compiler should just fall back to fetching it
individually.
Which Deltas To Create?
=======================
Going back to the start, there are two cases to optimize for:
1) Incremental upgrades between builds
2) Major version upgrades
A command line operation would look something like this:
$ ostree --repo=/path/to/repo gendelta --ref-prefix=gnome-ostree/buildmaster/ --strategy=latest --depth=5
This would tell ostree to generate deltas from each of the last 4
commits to each ref (e.g. gnome-ostree/buildmaster/x86_64-runtime) to
the latest commit. It might also be possible of course to have
--strategy=incremental where we generate a delta between each commit.
I suspect that'd be something to do if one has a *lot* of disk space
to spend, and there's a reason for clients to be fetching individual
refs.
$ ostree --repo=/path/to/repo gendelta --from=gnome-ostree/3.10/x86_64-runtime --to=gnome-ostree/buildmaster/x86_64-runtime
This is an obvious one - generate a delta from the last stable release
to the current development head.
src/libostree/ostree-bootloader-grub2.c
-2
View File
@@ -388,8 +388,6 @@ rm -f ${grub_cfg}.new
if (!g_file_copy (self->config_path_efi, config_path_efi_old,
G_FILE_COPY_OVERWRITE, cancellable, NULL, NULL, error))
goto out;
if (!ot_gfile_ensure_unlinked (config_path_efi_old, cancellable, error))
goto out;
/* NOTE: NON-ATOMIC REPLACEMENT; WE can't do anything else on FAT;
* see https://bugzilla.gnome.org/show_bug.cgi?id=724246
src/libostree/ostree-cmdprivate.c
+3 -1
View File
@@ -23,6 +23,7 @@
#include "ostree-cmdprivate.h"
#include "ostree-repo-private.h"
#include "ostree-core-private.h"
#include "ostree-repo-static-delta-private.h"
#include "ostree-sysroot.h"
#include "ostree-bootloader-grub2.h"
@@ -44,7 +45,8 @@ const OstreeCmdPrivateVTable *
ostree_cmd__private__ (void)
{
static OstreeCmdPrivateVTable table = {
impl_ostree_generate_grub2_config
impl_ostree_generate_grub2_config,
_ostree_repo_static_delta_dump
};
return &table;
src/libostree/ostree-cmdprivate.h
+1
View File
@@ -26,6 +26,7 @@ G_BEGIN_DECLS
typedef struct {
gboolean (* ostree_generate_grub2_config) (OstreeSysroot *sysroot, int bootversion, int target_fd, GCancellable *cancellable, GError **error);
gboolean (* ostree_static_delta_dump) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error);
} OstreeCmdPrivateVTable;
const OstreeCmdPrivateVTable *
src/libostree/ostree-core.c
+12 -12
View File
@@ -1069,7 +1069,7 @@ int
ostree_cmp_checksum_bytes (const guchar *a,
const guchar *b)
{
return memcmp (a, b, 32);
return memcmp (a, b, OSTREE_SHA256_DIGEST_LEN);
}
/**
@@ -1151,7 +1151,7 @@ ostree_checksum_inplace_to_bytes (const char *checksum,
guint i;
guint j;
for (i = 0, j = 0; i < 32; i += 1, j += 2)
for (i = 0, j = 0; i < OSTREE_SHA256_DIGEST_LEN; i += 1, j += 2)
{
gint big, little;
@@ -1177,7 +1177,7 @@ ostree_checksum_inplace_to_bytes (const char *checksum,
guchar *
ostree_checksum_to_bytes (const char *checksum)
{
guchar *ret = g_malloc (32);
guchar *ret = g_malloc (OSTREE_SHA256_DIGEST_LEN);
ostree_checksum_inplace_to_bytes (checksum, ret);
return ret;
}
@@ -1191,9 +1191,9 @@ ostree_checksum_to_bytes (const char *checksum)
GVariant *
ostree_checksum_to_bytes_v (const char *checksum)
{
guchar result[32];
guchar result[OSTREE_SHA256_DIGEST_LEN];
ostree_checksum_inplace_to_bytes (checksum, result);
return ot_gvariant_new_bytearray ((guchar*)result, 32);
return ot_gvariant_new_bytearray ((guchar*)result, OSTREE_SHA256_DIGEST_LEN);
}
/**
@@ -1210,7 +1210,7 @@ ostree_checksum_inplace_from_bytes (const guchar *csum,
static const gchar hexchars[] = "0123456789abcdef";
guint i, j;
for (i = 0, j = 0; i < 32; i++, j += 2)
for (i = 0, j = 0; i < OSTREE_SHA256_DIGEST_LEN; i++, j += 2)
{
guchar byte = csum[i];
buf[j] = hexchars[byte >> 4];
@@ -1242,7 +1242,7 @@ ostree_checksum_b64_inplace_from_bytes (const guchar *csum,
* a lot easier to reuse GLib's base64 encoder and postprocess it
* to replace the '/' with '_'.
*/
outlen = g_base64_encode_step (csum, 32, FALSE, tmpbuf, &state, &save);
outlen = g_base64_encode_step (csum, OSTREE_SHA256_DIGEST_LEN, FALSE, tmpbuf, &state, &save);
outlen += g_base64_encode_close (FALSE, tmpbuf+outlen, &state, &save);
g_assert (outlen == 44);
@@ -1299,7 +1299,7 @@ ostree_checksum_bytes_peek (GVariant *bytes)
gsize n_elts;
const guchar *ret;
ret = g_variant_get_fixed_array (bytes, &n_elts, 1);
if (G_UNLIKELY (n_elts != 32))
if (G_UNLIKELY (n_elts != OSTREE_SHA256_DIGEST_LEN))
return NULL;
return ret;
}
@@ -1434,20 +1434,20 @@ _ostree_get_relative_static_delta_path (const char *from,
const char *to,
const char *target)
{
guint8 csum_to[32];
guint8 csum_to[OSTREE_SHA256_DIGEST_LEN];
char to_b64[44];
guint8 csum_to_copy[32];
guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN];
GString *ret = g_string_new ("deltas/");
ostree_checksum_inplace_to_bytes (to, csum_to);
ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64);
ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy);
g_assert (memcmp (csum_to, csum_to_copy, 32) == 0);
g_assert (memcmp (csum_to, csum_to_copy, OSTREE_SHA256_DIGEST_LEN) == 0);
if (from != NULL)
{
guint8 csum_from[32];
guint8 csum_from[OSTREE_SHA256_DIGEST_LEN];
char from_b64[44];
ostree_checksum_inplace_to_bytes (from, csum_from);
src/libostree/ostree-core.h
+2
View File
@@ -50,6 +50,8 @@ G_BEGIN_DECLS
*/
#define OSTREE_MAX_RECURSION (256)
#define OSTREE_SHA256_DIGEST_LEN (32)
/**
* OstreeObjectType:
* @OSTREE_OBJECT_TYPE_FILE: Content; regular file, symbolic link
src/libostree/ostree-fetcher.c
+7 -2
View File
@@ -277,7 +277,12 @@ static void
session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure,
gpointer data)
{
GTlsInteraction *interaction = data;
GTlsCertificate *cert = data;
glnx_unref_object OstreeTlsCertInteraction *interaction = NULL;
/* The GTlsInteraction instance must be created in the
* session thread so it uses the correct GMainContext. */
interaction = _ostree_tls_cert_interaction_new (cert);
g_object_set (thread_closure->session,
SOUP_SESSION_TLS_INTERACTION,
@@ -645,7 +650,7 @@ _ostree_fetcher_set_client_cert (OstreeFetcher *self,
#ifdef HAVE_LIBSOUP_CLIENT_CERTS
session_thread_idle_add (self->thread_closure,
session_thread_set_tls_interaction_cb,
_ostree_tls_cert_interaction_new (cert),
g_object_ref (cert),
(GDestroyNotify) g_object_unref);
#else
g_warning ("This version of OSTree is compiled without client side certificate support");
src/libostree/ostree-repo-checkout.c
+60 -21
View File
@@ -32,6 +32,8 @@
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
#define WHITEOUT_PREFIX ".wh."
static gboolean
checkout_object_for_uncompressed_cache (OstreeRepo *self,
const char *loose_path,
@@ -102,9 +104,16 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self,
return ret;
}
static gboolean
fsync_is_enabled (OstreeRepo *self,
OstreeRepoCheckoutOptions *options)
{
return !(self->disable_fsync || options->disable_fsync);
}
static gboolean
write_regular_file_content (OstreeRepo *self,
OstreeRepoCheckoutMode mode,
OstreeRepoCheckoutOptions *options,
GOutputStream *output,
GFileInfo *file_info,
GVariant *xattrs,
@@ -113,6 +122,7 @@ write_regular_file_content (OstreeRepo *self,
GError **error)
{
gboolean ret = FALSE;
const OstreeRepoCheckoutMode mode = options->mode;
int fd;
int res;
@@ -154,12 +164,12 @@ write_regular_file_content (OstreeRepo *self,
}
}
if (!self->disable_fsync)
if (fsync_is_enabled (self, options))
{
if (fsync (fd) == -1)
{
gs_set_error_from_errno (error, errno);
goto out;
goto out;
}
}
@@ -238,7 +248,7 @@ checkout_file_from_input_at (OstreeRepo *self,
temp_out = g_unix_output_stream_new (fd, TRUE);
fd = -1; /* Transfer ownership */
if (!write_regular_file_content (self, options->mode, temp_out, file_info, xattrs, input,
if (!write_regular_file_content (self, options, temp_out, file_info, xattrs, input,
cancellable, error))
goto out;
}
@@ -298,7 +308,7 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo,
cancellable, error))
goto out;
if (!write_regular_file_content (repo, options->mode, temp_out, file_info, xattrs, input,
if (!write_regular_file_content (repo, options, temp_out, file_info, xattrs, input,
cancellable, error))
goto out;
}
@@ -388,20 +398,46 @@ checkout_one_file_at (OstreeRepo *repo,
const char *checksum;
gboolean is_symlink;
gboolean can_cache;
gboolean did_hardlink = FALSE;
gboolean need_copy = TRUE;
char loose_path_buf[_OSTREE_LOOSE_PATH_MAX];
g_autoptr(GInputStream) input = NULL;
g_autoptr(GVariant) xattrs = NULL;
gboolean is_whiteout;
is_symlink = g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK;
checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)source);
/* Try to do a hardlink first, if it's a regular file. This also
* traverses all parent repos.
is_whiteout = !is_symlink && options->process_whiteouts &&
g_str_has_prefix (destination_name, WHITEOUT_PREFIX);
/* First, see if it's a Docker whiteout,
* https://github.com/docker/docker/blob/1a714e76a2cb9008cd19609059e9988ff1660b78/pkg/archive/whiteouts.go
*/
if (!is_symlink)
if (is_whiteout)
{
const char *name = destination_name + (sizeof (WHITEOUT_PREFIX) - 1);
if (!name[0])
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid empty whiteout '%s'", name);
goto out;
}
g_assert (name[0] != '/'); /* Sanity */
if (!glnx_shutil_rm_rf_at (destination_dfd, name, cancellable, error))
goto out;
need_copy = FALSE;
}
else if (!is_symlink)
{
gboolean did_hardlink = FALSE;
/* Try to do a hardlink first, if it's a regular file. This also
* traverses all parent repos.
*/
OstreeRepo *current_repo = repo;
while (current_repo)
@@ -454,6 +490,8 @@ checkout_one_file_at (OstreeRepo *repo,
}
current_repo = current_repo->parent_repo;
}
need_copy = !did_hardlink;
}
can_cache = (options->enable_uncompressed_cache
@@ -463,11 +501,14 @@ checkout_one_file_at (OstreeRepo *repo,
* it now, stick it in the cache, and then hardlink to that.
*/
if (can_cache
&& !is_whiteout
&& !is_symlink
&& !did_hardlink
&& need_copy
&& repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2
&& options->mode == OSTREE_REPO_CHECKOUT_MODE_USER)
{
gboolean did_hardlink;
if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL,
cancellable, error))
goto out;
@@ -518,10 +559,12 @@ checkout_one_file_at (OstreeRepo *repo,
g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name);
goto out;
}
need_copy = !did_hardlink;
}
/* Fall back to copy if we couldn't hardlink */
if (!did_hardlink)
if (need_copy)
{
if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs,
cancellable, error))
@@ -726,17 +769,13 @@ checkout_tree_at (OstreeRepo *self,
}
}
/* Finally, fsync to ensure all entries are on disk. Ultimately
* this should be configurable for the case where we're constructing
* buildroots.
*/
if (!self->disable_fsync)
if (fsync_is_enabled (self, options))
{
if (fsync (destination_dfd) == -1)
{
gs_set_error_from_errno (error, errno);
goto out;
}
if (fsync (destination_dfd) == -1)
{
gs_set_error_from_errno (error, errno);
goto out;
}
}
ret = TRUE;
src/libostree/ostree-repo-commit.c
+4 -3
View File
@@ -412,12 +412,12 @@ add_size_index_to_metadata (OstreeRepo *self,
for (i = 0; i < sorted_keys->len; i++)
{
guint8 csum[32];
guint8 csum[OSTREE_SHA256_DIGEST_LEN];
const char *e_checksum = sorted_keys->pdata[i];
GString *buffer = g_string_new (NULL);
ostree_checksum_inplace_to_bytes (e_checksum, csum);
g_string_append_len (buffer, (char*)csum, 32);
g_string_append_len (buffer, (char*)csum, sizeof (csum));
e_size = g_hash_table_lookup (self->object_sizes, e_checksum);
_ostree_write_varuint64 (buffer, e_size->archived);
@@ -1144,7 +1144,8 @@ ostree_repo_scan_hardlinks (OstreeRepo *self,
* ostree_repo_prepare_transaction:
* @self: An #OstreeRepo
* @out_transaction_resume: (allow-none) (out): Whether this transaction
* is resuming from a previous one.
* is resuming from a previous one. This is a legacy state, now OSTree
* pulls use per-commit `state/.commitpartial` files.
* @cancellable: Cancellable
* @error: Error
*
src/libostree/ostree-repo-libarchive.c
+390 -40
View File
@@ -24,6 +24,7 @@
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
#include "ostree-repo-file.h"
#include "ostree-mutable-tree.h"
#ifdef HAVE_LIBARCHIVE
@@ -77,6 +78,7 @@ file_info_from_archive_entry_and_modifier (OstreeRepo *repo,
static gboolean
import_libarchive_entry_file (OstreeRepo *self,
OstreeRepoImportArchiveOptions *opts,
struct archive *a,
struct archive_entry *entry,
GFileInfo *file_info,
@@ -92,8 +94,27 @@ import_libarchive_entry_file (OstreeRepo *self,
if (g_cancellable_set_error_if_cancelled (cancellable, error))
return FALSE;
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
archive_stream = _ostree_libarchive_input_stream_new (a);
switch (g_file_info_get_file_type (file_info))
{
case G_FILE_TYPE_REGULAR:
archive_stream = _ostree_libarchive_input_stream_new (a);
break;
case G_FILE_TYPE_SYMBOLIC_LINK:
break;
default:
if (opts->ignore_unsupported_content)
{
ret = TRUE;
goto out;
}
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Unable to import non-regular/non-symlink file '%s'",
archive_entry_pathname (entry));
goto out;
}
}
if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL,
&file_object_input, &length, cancellable, error))
@@ -110,6 +131,7 @@ import_libarchive_entry_file (OstreeRepo *self,
static gboolean
write_libarchive_entry_to_mtree (OstreeRepo *self,
OstreeRepoImportArchiveOptions *opts,
OstreeMutableTree *root,
struct archive *a,
struct archive_entry *entry,
@@ -248,16 +270,19 @@ write_libarchive_entry_to_mtree (OstreeRepo *self,
goto out;
}
if (!import_libarchive_entry_file (self, a, entry, file_info, &tmp_csum,
if (!import_libarchive_entry_file (self, opts, a, entry, file_info, &tmp_csum,
cancellable, error))
goto out;
g_free (tmp_checksum);
tmp_checksum = ostree_checksum_from_bytes (tmp_csum);
if (!ostree_mutable_tree_replace_file (parent, basename,
tmp_checksum,
error))
goto out;
if (tmp_csum)
{
g_free (tmp_checksum);
tmp_checksum = ostree_checksum_from_bytes (tmp_csum);
if (!ostree_mutable_tree_replace_file (parent, basename,
tmp_checksum,
error))
goto out;
}
}
}
@@ -266,6 +291,109 @@ write_libarchive_entry_to_mtree (OstreeRepo *self,
return ret;
}
#endif
static gboolean
create_empty_dir_with_uidgid (OstreeRepo *self,
guint32 uid,
guint32 gid,
guint8 **out_csum,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new ();
g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", uid);
g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", gid);
g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR);
return _ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, out_csum, cancellable, error);
}
/**
* ostree_repo_import_archive_to_mtree:
* @self: An #OstreeRepo
* @opts: Options structure, ensure this is zeroed, then set specific variables
* @archive: Really this is "struct archive*"
* @mtree: The #OstreeMutableTree to write to
* @modifier: (allow-none): Optional commit modifier
* @cancellable: Cancellable
* @error: Error
*
* Import an archive file @archive into the repository, and write its
* file structure to @mtree.
*/
gboolean
ostree_repo_import_archive_to_mtree (OstreeRepo *self,
OstreeRepoImportArchiveOptions *opts,
void *archive,
OstreeMutableTree *mtree,
OstreeRepoCommitModifier *modifier,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
struct archive *a = archive;
struct archive_entry *entry;
g_autofree guchar *tmp_csum = NULL;
int r;
while (TRUE)
{
r = archive_read_next_header (a, &entry);
if (r == ARCHIVE_EOF)
break;
else if (r != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
/* TODO - refactor this to only create the metadata on demand
* (i.e. if there is a missing parent dir)
*/
if (opts->autocreate_parents && !tmp_csum)
{
/* Here, we auto-pick the first uid/gid we find in the
* archive. Realistically this is probably always going to
* be root, but eh, at least we try to match.
*/
if (!create_empty_dir_with_uidgid (self, archive_entry_uid (entry),
archive_entry_gid (entry),
&tmp_csum, cancellable, error))
goto out;
}
if (!write_libarchive_entry_to_mtree (self, opts, mtree, a,
entry, modifier, tmp_csum,
cancellable, error))
goto out;
}
/* If we didn't import anything at all, and autocreation of parents
* is enabled, automatically create a root directory. This is
* useful primarily when importing Docker image layers, which can
* just be metadata.
*/
if (!ostree_mutable_tree_get_metadata_checksum (mtree) && opts->autocreate_parents)
{
char tmp_checksum[65];
if (!tmp_csum)
{
/* We didn't have any archive entries to match, so pick uid 0, gid 0. */
if (!create_empty_dir_with_uidgid (self, 0, 0, &tmp_csum, cancellable, error))
goto out;
}
ostree_checksum_inplace_from_bytes (tmp_csum, tmp_checksum);
ostree_mutable_tree_set_metadata_checksum (mtree, tmp_checksum);
}
ret = TRUE;
out:
return ret;
}
/**
* ostree_repo_write_archive_to_mtree:
@@ -292,10 +420,8 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self,
#ifdef HAVE_LIBARCHIVE
gboolean ret = FALSE;
struct archive *a = NULL;
struct archive_entry *entry;
int r;
g_autoptr(GFileInfo) tmp_dir_info = NULL;
g_autofree guchar *tmp_csum = NULL;
OstreeRepoImportArchiveOptions opts = { 0, };
a = archive_read_new ();
#ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
@@ -310,35 +436,11 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self,
goto out;
}
while (TRUE)
{
r = archive_read_next_header (a, &entry);
if (r == ARCHIVE_EOF)
break;
else if (r != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
opts.autocreate_parents = !!autocreate_parents;
if (autocreate_parents && !tmp_csum)
{
tmp_dir_info = g_file_info_new ();
g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry));
g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry));
g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR);
if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error))
goto out;
}
if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error))
goto out;
if (!write_libarchive_entry_to_mtree (self, mtree, a,
entry, modifier,
autocreate_parents ? tmp_csum : NULL,
cancellable, error))
goto out;
}
if (archive_read_close (a) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
@@ -356,3 +458,251 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self,
return FALSE;
#endif
}
#ifdef HAVE_LIBARCHIVE
static gboolean
file_to_archive_entry_common (GFile *root,
OstreeRepoExportArchiveOptions *opts,
GFile *path,
GFileInfo *file_info,
struct archive_entry *entry,
GError **error)
{
gboolean ret = FALSE;
g_autofree char *pathstr = g_file_get_relative_path (root, path);
g_autoptr(GVariant) xattrs = NULL;
time_t ts = (time_t) opts->timestamp_secs;
if (pathstr && !pathstr[0])
{
g_free (pathstr);
pathstr = g_strdup (".");
}
archive_entry_update_pathname_utf8 (entry, pathstr);
archive_entry_set_ctime (entry, ts, 0);
archive_entry_set_mtime (entry, ts, 0);
archive_entry_set_atime (entry, ts, 0);
archive_entry_set_uid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::uid"));
archive_entry_set_gid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::gid"));
archive_entry_set_mode (entry, g_file_info_get_attribute_uint32 (file_info, "unix::mode"));
if (!ostree_repo_file_get_xattrs ((OstreeRepoFile*)path, &xattrs, NULL, error))
goto out;
if (!opts->disable_xattrs)
{
int i, n;
n = g_variant_n_children (xattrs);
for (i = 0; i < n; i++)
{
const guint8* name;
g_autoptr(GVariant) value = NULL;
const guint8* value_data;
gsize value_len;
g_variant_get_child (xattrs, i, "(^&ay@ay)", &name, &value);
value_data = g_variant_get_fixed_array (value, &value_len, 1);
archive_entry_xattr_add_entry (entry, (char*)name,
(char*) value_data, value_len);
}
}
ret = TRUE;
out:
return ret;
}
static gboolean
write_header_free_entry (struct archive *a,
struct archive_entry **entryp,
GError **error)
{
struct archive_entry *entry = *entryp;
gboolean ret = FALSE;
if (archive_write_header (a, entry) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
ret = TRUE;
out:
archive_entry_free (entry);
*entryp = NULL;
return ret;
}
static gboolean
write_directory_to_libarchive_recurse (OstreeRepo *self,
OstreeRepoExportArchiveOptions *opts,
GFile *root,
GFile *dir,
struct archive *a,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GFileInfo) dir_info = NULL;
g_autoptr(GFileEnumerator) dir_enum = NULL;
struct archive_entry *entry = NULL;
dir_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!dir_info)
goto out;
entry = archive_entry_new2 (a);
if (!file_to_archive_entry_common (root, opts, dir, dir_info, entry, error))
goto out;
if (!write_header_free_entry (a, &entry, error))
goto out;
dir_enum = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!dir_enum)
goto out;
while (TRUE)
{
GFileInfo *file_info;
GFile *path;
if (!gs_file_enumerator_iterate (dir_enum, &file_info, &path,
cancellable, error))
goto out;
if (file_info == NULL)
break;
/* First, handle directories recursively */
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
{
if (!write_directory_to_libarchive_recurse (self, opts, root, path, a,
cancellable, error))
goto out;
/* Go to the next entry */
continue;
}
/* Past here, should be a regular file or a symlink */
entry = archive_entry_new2 (a);
if (!file_to_archive_entry_common (root, opts, path, file_info, entry, error))
goto out;
switch (g_file_info_get_file_type (file_info))
{
case G_FILE_TYPE_SYMBOLIC_LINK:
{
archive_entry_set_symlink (entry, g_file_info_get_symlink_target (file_info));
if (!write_header_free_entry (a, &entry, error))
goto out;
}
break;
case G_FILE_TYPE_REGULAR:
{
guint8 buf[8192];
g_autoptr(GInputStream) file_in = NULL;
g_autoptr(GFileInfo) file_info = NULL;
const char *checksum;
checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)path);
if (!ostree_repo_load_file (self, checksum, &file_in, &file_info, NULL,
cancellable, error))
goto out;
archive_entry_set_size (entry, g_file_info_get_size (file_info));
if (archive_write_header (a, entry) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
while (TRUE)
{
gssize bytes_read = g_input_stream_read (file_in, buf, sizeof (buf),
cancellable, error);
if (bytes_read < 0)
goto out;
if (bytes_read == 0)
break;
{ ssize_t r = archive_write_data (a, buf, bytes_read);
if (r != bytes_read)
{
propagate_libarchive_error (error, a);
g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", (guint64)bytes_read, (guint64)r);
goto out;
}
}
}
if (archive_write_finish_entry (a) != ARCHIVE_OK)
{
propagate_libarchive_error (error, a);
goto out;
}
archive_entry_free (entry);
entry = NULL;
}
break;
default:
g_assert_not_reached ();
}
}
ret = TRUE;
out:
if (entry)
archive_entry_free (entry);
return ret;
}
#endif
/**
* ostree_repo_export_tree_to_archive:
* @self: An #OstreeRepo
* @opts: Options controlling conversion
* @root: An #OstreeRepoFile for the base directory
* @archive: A `struct archive`, but specified as void to avoid a dependency on the libarchive headers
* @cancellable: Cancellable
* @error: Error
*
* Import an archive file @archive into the repository, and write its
* file structure to @mtree.
*/
gboolean
ostree_repo_export_tree_to_archive (OstreeRepo *self,
OstreeRepoExportArchiveOptions *opts,
OstreeRepoFile *root,
void *archive,
GCancellable *cancellable,
GError **error)
{
#ifdef HAVE_LIBARCHIVE
gboolean ret = FALSE;
struct archive *a = archive;
if (!write_directory_to_libarchive_recurse (self, opts, (GFile*)root, (GFile*)root,
a, cancellable, error))
goto out;
ret = TRUE;
out:
return ret;
#else
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of ostree is not compiled with libarchive support");
return FALSE;
#endif
}
src/libostree/ostree-repo-private.h
-2
View File
@@ -50,8 +50,6 @@ struct OstreeRepo {
int repo_dir_fd;
GFile *tmp_dir;
int tmp_dir_fd;
GFile *local_heads_dir;
GFile *remote_heads_dir;
GFile *objects_dir;
GFile *state_dir;
int objects_dir_fd;
src/libostree/ostree-repo-pull.c
+161 -107
View File
@@ -48,7 +48,9 @@ typedef struct {
GCancellable *cancellable;
OstreeAsyncProgress *progress;
gboolean transaction_resuming;
gboolean dry_run;
gboolean dry_run_emitted_progress;
gboolean legacy_transaction_resuming;
enum {
OSTREE_PULL_PHASE_FETCHING_REFS,
OSTREE_PULL_PHASE_FETCHING_OBJECTS
@@ -78,6 +80,7 @@ typedef struct {
guint n_outstanding_deltapart_write_requests;
guint n_total_deltaparts;
guint64 total_deltapart_size;
guint64 total_deltapart_usize;
gint n_requested_metadata;
gint n_requested_content;
guint n_fetched_deltaparts;
@@ -123,7 +126,7 @@ typedef struct {
} FetchStaticDeltaData;
typedef struct {
guchar csum[32];
guchar csum[OSTREE_SHA256_DIGEST_LEN];
OstreeObjectType objtype;
guint recursion_depth;
} ScanObjectQueueData;
@@ -227,6 +230,8 @@ update_progress (gpointer user_data)
pull_data->n_total_deltaparts);
ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size",
pull_data->total_deltapart_size);
ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize",
pull_data->total_deltapart_usize);
ostree_async_progress_set_uint (pull_data->progress, "total-delta-superblocks",
pull_data->static_delta_superblocks->len);
@@ -243,6 +248,9 @@ update_progress (gpointer user_data)
else
ostree_async_progress_set_status (pull_data->progress, NULL);
if (pull_data->dry_run)
pull_data->dry_run_emitted_progress = TRUE;
return TRUE;
}
@@ -262,6 +270,9 @@ pull_termination_condition (OtPullData *pull_data)
if (pull_data->caught_error)
return TRUE;
if (pull_data->dry_run)
return pull_data->dry_run_emitted_progress;
switch (pull_data->phase)
{
case OSTREE_PULL_PHASE_FETCHING_REFS:
@@ -935,8 +946,7 @@ static_deltapart_fetch_on_complete (GObject *object,
g_autoptr(GVariant) metadata = NULL;
g_autofree char *temp_path = NULL;
g_autoptr(GInputStream) in = NULL;
g_autofree char *actual_checksum = NULL;
g_autofree guint8 *csum = NULL;
g_autoptr(GVariant) part = NULL;
GError *local_error = NULL;
GError **error = &local_error;
gs_fd_close int fd = -1;
@@ -950,54 +960,33 @@ static_deltapart_fetch_on_complete (GObject *object,
fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC);
if (fd == -1)
{
gs_set_error_from_errno (error, errno);
glnx_set_error_from_errno (error);
goto out;
}
/* From here on, if we fail to apply the delta, we'll re-fetch it */
if (unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0) < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
in = g_unix_input_stream_new (fd, FALSE);
/* TODO - consider making async */
if (!ot_gio_checksum_stream (in, &csum, pull_data->cancellable, error))
/* TODO - make async */
if (!_ostree_static_delta_part_open (in, NULL, 0, fetch_data->expected_checksum,
&part, pull_data->cancellable, error))
goto out;
actual_checksum = ostree_checksum_from_bytes (csum);
if (strcmp (actual_checksum, fetch_data->expected_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted static delta part; checksum expected='%s' actual='%s'",
fetch_data->expected_checksum, actual_checksum);
goto out;
}
/* Might as well close the fd here */
(void) g_input_stream_close (in, NULL, NULL);
{
GMappedFile *mfile = NULL;
g_autoptr(GBytes) delta_data = NULL;
mfile = g_mapped_file_new_from_fd (fd, FALSE, error);
if (!mfile)
goto out;
delta_data = g_mapped_file_get_bytes (mfile);
g_mapped_file_unref (mfile);
/* Unlink now while we're holding an open fd, so that on success
* or error, the file will be gone. This is particularly
* important if say we hit e.g. ENOSPC.
*/
(void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0);
_ostree_static_delta_part_execute_async (pull_data->repo,
fetch_data->objects,
delta_data,
/* Trust checksums if summary was gpg signed */
pull_data->gpg_verify_summary && pull_data->summary_data_sig,
pull_data->cancellable,
on_static_delta_written,
fetch_data);
pull_data->n_outstanding_deltapart_write_requests++;
}
_ostree_static_delta_part_execute_async (pull_data->repo,
fetch_data->objects,
part,
/* Trust checksums if summary was gpg signed */
pull_data->gpg_verify_summary && pull_data->summary_data_sig,
pull_data->cancellable,
on_static_delta_written,
fetch_data);
pull_data->n_outstanding_deltapart_write_requests++;
out:
g_assert (pull_data->n_outstanding_deltapart_fetches > 0);
@@ -1150,7 +1139,7 @@ queue_scan_one_metadata_object (OtPullData *pull_data,
OstreeObjectType objtype,
guint recursion_depth)
{
guchar buf[32];
guchar buf[OSTREE_SHA256_DIGEST_LEN];
ostree_checksum_inplace_to_bytes (csum, buf);
queue_scan_one_metadata_object_c (pull_data, buf, objtype, recursion_depth);
}
@@ -1227,7 +1216,7 @@ scan_one_metadata_object_c (OtPullData *pull_data,
}
else if (is_stored)
{
gboolean do_scan = pull_data->transaction_resuming || is_requested || pull_data->commitpartial_exists;
gboolean do_scan = pull_data->legacy_transaction_resuming || is_requested || pull_data->commitpartial_exists;
/* For commits, always refetch detached metadata. */
if (objtype == OSTREE_OBJECT_TYPE_COMMIT)
@@ -1454,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data,
static gboolean
process_one_static_delta_fallback (OtPullData *pull_data,
gboolean delta_byteswap,
GVariant *fallback_object,
GCancellable *cancellable,
GError **error)
@@ -1473,10 +1463,20 @@ process_one_static_delta_fallback (OtPullData *pull_data,
if (!ostree_validate_structureof_csum_v (csum_v, error))
goto out;
objtype = (OstreeObjectType)objtype_y;
checksum = ostree_checksum_from_bytes_v (csum_v);
compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size);
uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size);
pull_data->total_deltapart_size += compressed_size;
pull_data->total_deltapart_usize += uncompressed_size;
if (pull_data->dry_run)
{
ret = TRUE;
goto out;
}
objtype = (OstreeObjectType)objtype_y;
checksum = ostree_checksum_from_bytes_v (csum_v);
if (!ostree_repo_has_object (pull_data->repo, objtype, checksum,
&is_stored,
@@ -1522,11 +1522,14 @@ process_one_static_delta (OtPullData *pull_data,
GError **error)
{
gboolean ret = FALSE;
gboolean delta_byteswap;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) headers = NULL;
g_autoptr(GVariant) fallback_objects = NULL;
guint i, n;
delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock);
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
metadata = g_variant_get_child_value (delta_superblock, 0);
headers = g_variant_get_child_value (delta_superblock, 6);
@@ -1539,13 +1542,14 @@ process_one_static_delta (OtPullData *pull_data,
g_autoptr(GVariant) fallback_object =
g_variant_get_child_value (fallback_objects, i);
if (!process_one_static_delta_fallback (pull_data,
if (!process_one_static_delta_fallback (pull_data, delta_byteswap,
fallback_object,
cancellable, error))
goto out;
}
/* Write the to-commit object */
if (!pull_data->dry_run)
{
g_autoptr(GVariant) to_csum_v = NULL;
g_autofree char *to_checksum = NULL;
@@ -1604,14 +1608,18 @@ process_one_static_delta (OtPullData *pull_data,
FetchStaticDeltaData *fetch_data;
g_autoptr(GVariant) csum_v = NULL;
g_autoptr(GVariant) objects = NULL;
g_autoptr(GVariant) part_data = NULL;
g_autoptr(GBytes) delta_data = NULL;
g_autoptr(GBytes) inline_part_bytes = NULL;
guint64 size, usize;
guint32 version;
const gboolean trusted = pull_data->gpg_verify_summary && pull_data->summary_data_sig;
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
version = maybe_swap_endian_u32 (delta_byteswap, version);
size = maybe_swap_endian_u64 (delta_byteswap, size);
usize = maybe_swap_endian_u64 (delta_byteswap, usize);
if (version > OSTREE_DELTAPART_VERSION)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
@@ -1623,31 +1631,6 @@ process_one_static_delta (OtPullData *pull_data,
if (!csum)
goto out;
deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)"));
if (part_data)
{
g_autofree char *actual_checksum = NULL;
g_autofree char *expected_checksum = ostree_checksum_from_bytes_v (csum_v);
delta_data = g_variant_get_data_as_bytes (part_data);
/* For inline parts we are relying on per-commit GPG, so this isn't strictly necessary for security.
* See https://github.com/GNOME/ostree/pull/139
*/
actual_checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, delta_data);
if (strcmp (actual_checksum, expected_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted static delta part; checksum expected='%s' actual='%s'",
expected_checksum, actual_checksum);
goto out;
}
}
pull_data->total_deltapart_size += size;
if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo,
objects,
&have_all,
@@ -1663,18 +1646,42 @@ process_one_static_delta (OtPullData *pull_data,
continue;
}
deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i);
{ g_autoptr(GVariant) part_datav =
g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)"));
if (part_datav)
inline_part_bytes = g_variant_get_data_as_bytes (part_datav);
}
pull_data->total_deltapart_size += size;
pull_data->total_deltapart_usize += usize;
if (pull_data->dry_run)
continue;
fetch_data = g_new0 (FetchStaticDeltaData, 1);
fetch_data->pull_data = pull_data;
fetch_data->objects = g_variant_ref (objects);
fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
if (delta_data != NULL)
if (inline_part_bytes != NULL)
{
g_autoptr(GInputStream) memin = g_memory_input_stream_new_from_bytes (inline_part_bytes);
g_autoptr(GVariant) inline_delta_part = NULL;
/* For inline parts we are relying on per-commit GPG, so don't bother checksumming. */
if (!_ostree_static_delta_part_open (memin, inline_part_bytes,
OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM,
NULL, &inline_delta_part,
cancellable, error))
goto out;
_ostree_static_delta_part_execute_async (pull_data->repo,
fetch_data->objects,
delta_data,
/* Trust checksums if summary was gpg signed */
pull_data->gpg_verify_summary && pull_data->summary_data_sig,
inline_delta_part,
trusted,
pull_data->cancellable,
on_static_delta_written,
fetch_data);
@@ -1789,8 +1796,10 @@ ostree_repo_pull_with_options (OstreeRepo *self,
OstreeRepoPullFlags flags = 0;
const char *dir_to_pull = NULL;
char **refs_to_fetch = NULL;
char **override_commit_ids = NULL;
GSource *update_timeout = NULL;
gboolean disable_static_deltas = FALSE;
gboolean require_static_deltas = FALSE;
if (options)
{
@@ -1803,13 +1812,24 @@ ostree_repo_pull_with_options (OstreeRepo *self,
(void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_name);
(void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth);
(void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas);
(void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas);
(void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids);
(void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run);
}
g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
if (refs_to_fetch && override_commit_ids)
g_return_val_if_fail (g_strv_length (refs_to_fetch) == g_strv_length (override_commit_ids), FALSE);
if (dir_to_pull)
g_return_val_if_fail (dir_to_pull[0] == '/', FALSE);
g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE);
/* We only do dry runs with static deltas, because we don't really have any
* in-advance information for bare fetches.
*/
g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE);
pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0;
pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0;
@@ -1992,6 +2012,13 @@ ostree_repo_pull_with_options (OstreeRepo *self,
goto out;
}
if (!bytes_summary && require_static_deltas)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Fetch configured to require static deltas, but no summary found");
goto out;
}
if (bytes_summary)
{
uri = suburi_new (pull_data->base_uri, "summary.sig", NULL);
@@ -2067,7 +2094,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
{
const char *delta;
GVariant *csum_v = NULL;
guchar *csum_data = g_malloc (32);
guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN);
g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i);
g_variant_get_child (ref, 0, "&s", &delta);
@@ -2096,8 +2123,10 @@ ostree_repo_pull_with_options (OstreeRepo *self,
}
else if (refs_to_fetch != NULL)
{
char **strviter;
for (strviter = refs_to_fetch; *strviter; strviter++)
char **strviter = refs_to_fetch;
char **commitid_strviter = override_commit_ids ? override_commit_ids : NULL;
while (*strviter)
{
const char *branch = *strviter;
@@ -2108,8 +2137,13 @@ ostree_repo_pull_with_options (OstreeRepo *self,
}
else
{
g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL);
char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL;
g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), commitid);
}
strviter++;
if (commitid_strviter)
commitid_strviter++;
}
}
else
@@ -2136,28 +2170,36 @@ ostree_repo_pull_with_options (OstreeRepo *self,
while (g_hash_table_iter_next (&hash_iter, &key, &value))
{
const char *branch = key;
const char *override_commitid = value;
char *contents = NULL;
if (pull_data->summary)
/* Support specifying "" for an override commitid */
if (override_commitid && *override_commitid)
{
gsize commit_size = 0;
guint64 *malloced_size;
if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error))
goto out;
malloced_size = g_new0 (guint64, 1);
*malloced_size = commit_size;
g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size);
g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), g_strdup (override_commitid));
}
else
else
{
if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
goto out;
if (pull_data->summary)
{
gsize commit_size = 0;
guint64 *malloced_size;
if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error))
goto out;
malloced_size = g_new0 (guint64, 1);
*malloced_size = commit_size;
g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size);
}
else
{
if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error))
goto out;
}
/* Transfer ownership of contents */
g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents);
}
/* Transfer ownership of contents */
g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents);
}
/* Create the state directory here - it's new with the commitpartial code,
@@ -2182,11 +2224,12 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (pull_data->fetcher == NULL)
goto out;
if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->transaction_resuming,
if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->legacy_transaction_resuming,
cancellable, error))
goto out;
g_debug ("resuming transaction: %s", pull_data->transaction_resuming ? "true" : " false");
if (pull_data->legacy_transaction_resuming)
g_debug ("resuming legacy transaction");
g_hash_table_iter_init (&hash_iter, commits_to_fetch);
while (g_hash_table_iter_next (&hash_iter, &key, &value))
@@ -2207,17 +2250,22 @@ ostree_repo_pull_with_options (OstreeRepo *self,
&from_revision, error))
goto out;
#ifdef BUILDOPT_STATIC_DELTAS
if (!disable_static_deltas && (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0))
{
if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision,
&delta_superblock, cancellable, error))
goto out;
}
#endif
if (!delta_superblock)
{
if (require_static_deltas)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Static deltas required, but none found for %s to %s",
from_revision, to_revision);
goto out;
}
g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision);
queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, 0);
}
@@ -2234,7 +2282,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (pull_data->progress)
{
update_timeout = g_timeout_source_new_seconds (1);
update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1);
g_source_set_priority (update_timeout, G_PRIORITY_HIGH);
g_source_set_callback (update_timeout, update_progress, pull_data, NULL);
g_source_attach (update_timeout, pull_data->main_context);
@@ -2247,6 +2295,12 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (pull_data->caught_error)
goto out;
if (pull_data->dry_run)
{
ret = TRUE;
goto out;
}
g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0);
g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0);
src/libostree/ostree-repo-refs.c
+156 -95
View File
@@ -25,19 +25,19 @@
static gboolean
add_ref_to_set (const char *remote,
GFile *base,
GFile *child,
int base_fd,
const char *path,
GHashTable *refs,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
char *contents;
char *relpath;
gsize len;
GString *refname;
if (!g_file_load_contents (child, cancellable, &contents, &len, NULL, error))
contents = glnx_file_get_contents_utf8_at (base_fd, path, &len, cancellable, error);
if (!contents)
goto out;
g_strchomp (contents);
@@ -48,9 +48,7 @@ add_ref_to_set (const char *remote,
g_string_append (refname, remote);
g_string_append_c (refname, ':');
}
relpath = g_file_get_relative_path (base, child);
g_string_append (refname, relpath);
g_free (relpath);
g_string_append (refname, path);
g_hash_table_insert (refs, g_string_free (refname, FALSE), contents);
@@ -116,44 +114,69 @@ write_checksum_file_at (OstreeRepo *self,
return ret;
}
static gboolean
openat_ignore_enoent (int dfd,
const char *path,
int *out_fd,
GError **error)
{
gboolean ret = FALSE;
int target_fd = -1;
target_fd = openat (dfd, path, O_CLOEXEC | O_RDONLY);
if (target_fd < 0)
{
if (errno != ENOENT)
{
glnx_set_error_from_errno (error);
goto out;
}
}
ret = TRUE;
*out_fd = target_fd;
out:
return ret;
}
static gboolean
find_ref_in_remotes (OstreeRepo *self,
const char *rev,
GFile **out_file,
int *out_fd,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GFileEnumerator) dir_enum = NULL;
g_autoptr(GFile) ret_file = NULL;
g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
glnx_fd_close int ret_fd = -1;
dir_enum = g_file_enumerate_children (self->remote_heads_dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
NULL, error);
if (!dir_enum)
if (!glnx_dirfd_iterator_init_at (self->repo_dir_fd, "refs/remotes", TRUE, &dfd_iter, error))
goto out;
while (TRUE)
{
GFileInfo *file_info;
GFile *child;
if (!gs_file_enumerator_iterate (dir_enum, &file_info, &child,
NULL, error))
struct dirent *dent = NULL;
glnx_fd_close int remote_dfd = -1;
if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, NULL, error))
goto out;
if (file_info == NULL)
if (dent == NULL)
break;
if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_DIRECTORY)
if (dent->d_type != DT_DIR)
continue;
g_clear_object (&ret_file);
ret_file = g_file_resolve_relative_path (child, rev);
if (!g_file_query_exists (ret_file, NULL))
g_clear_object (&ret_file);
else
if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &remote_dfd, error))
goto out;
if (!openat_ignore_enoent (remote_dfd, rev, &ret_fd, error))
goto out;
if (ret_fd != -1)
break;
}
ret = TRUE;
ot_transfer_out_value (out_file, &ret_file);
*out_fd = ret_fd; ret_fd = -1;
out:
return ret;
}
@@ -210,9 +233,8 @@ resolve_refspec (OstreeRepo *self,
{
gboolean ret = FALSE;
__attribute__((unused)) GCancellable *cancellable = NULL;
GError *temp_error = NULL;
g_autofree char *ret_rev = NULL;
g_autoptr(GFile) child = NULL;
glnx_fd_close int target_fd = -1;
g_return_val_if_fail (ref != NULL, FALSE);
@@ -223,40 +245,42 @@ resolve_refspec (OstreeRepo *self,
}
else if (remote != NULL)
{
child = ot_gfile_resolve_path_printf (self->remote_heads_dir, "%s/%s",
remote, ref);
if (!g_file_query_exists (child, NULL))
g_clear_object (&child);
const char *remote_ref = glnx_strjoina ("refs/remotes/", remote, "/", ref);
if (!openat_ignore_enoent (self->repo_dir_fd, remote_ref, &target_fd, error))
goto out;
}
else
{
child = g_file_resolve_relative_path (self->local_heads_dir, ref);
const char *local_ref = glnx_strjoina ("refs/heads/", ref);
if (!g_file_query_exists (child, NULL))
if (!openat_ignore_enoent (self->repo_dir_fd, local_ref, &target_fd, error))
goto out;
if (target_fd == -1)
{
g_clear_object (&child);
local_ref = glnx_strjoina ("refs/remotes/", ref);
child = g_file_resolve_relative_path (self->remote_heads_dir, ref);
if (!openat_ignore_enoent (self->repo_dir_fd, local_ref, &target_fd, error))
goto out;
if (!g_file_query_exists (child, NULL))
if (target_fd == -1)
{
g_clear_object (&child);
if (!find_ref_in_remotes (self, ref, &child, error))
if (!find_ref_in_remotes (self, ref, &target_fd, error))
goto out;
}
}
}
if (child)
if (target_fd != -1)
{
if ((ret_rev = gs_file_load_contents_utf8 (child, NULL, &temp_error)) == NULL)
ret_rev = glnx_fd_readall_utf8 (target_fd, NULL, NULL, error);
if (!ret_rev)
{
g_propagate_error (error, temp_error);
g_prefix_error (error, "Couldn't open ref '%s': ", gs_file_get_path_cached (child));
g_prefix_error (error, "Couldn't open ref '%s': ", ref);
goto out;
}
g_strchomp (ret_rev);
if (!ostree_validate_checksum_string (ret_rev, error))
goto out;
@@ -433,43 +457,50 @@ ostree_repo_resolve_rev (OstreeRepo *self,
static gboolean
enumerate_refs_recurse (OstreeRepo *repo,
const char *remote,
GFile *base,
GFile *dir,
int base_dfd,
GString *base_path,
int child_dfd,
const char *path,
GHashTable *refs,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GFileEnumerator) enumerator = NULL;
g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!enumerator)
if (!glnx_dirfd_iterator_init_at (child_dfd, path, FALSE, &dfd_iter, error))
goto out;
while (TRUE)
{
GFileInfo *file_info = NULL;
GFile *child = NULL;
guint len = base_path->len;
struct dirent *dent = NULL;
if (!gs_file_enumerator_iterate (enumerator, &file_info, &child,
NULL, error))
if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
goto out;
if (file_info == NULL)
if (dent == NULL)
break;
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY)
g_string_append (base_path, dent->d_name);
if (dent->d_type == DT_DIR)
{
if (!enumerate_refs_recurse (repo, remote, base, child, refs, cancellable, error))
g_string_append_c (base_path, '/');
if (!enumerate_refs_recurse (repo, remote, base_dfd, base_path,
dfd_iter.fd, dent->d_name,
refs, cancellable, error))
goto out;
}
else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
else if (dent->d_type == DT_REG)
{
if (!add_ref_to_set (remote, base, child, refs,
if (!add_ref_to_set (remote, base_dfd, base_path->str, refs,
cancellable, error))
goto out;
}
g_string_truncate (base_path, len);
}
ret = TRUE;
@@ -505,35 +536,55 @@ ostree_repo_list_refs (OstreeRepo *self,
if (refspec_prefix)
{
g_autoptr(GFile) dir = NULL;
g_autoptr(GFile) child = NULL;
g_autoptr(GFileInfo) info = NULL;
struct stat stbuf;
const char *prefix_path;
const char *path;
if (!ostree_parse_refspec (refspec_prefix, &remote, &ref_prefix, error))
goto out;
if (remote)
dir = g_file_get_child (self->remote_heads_dir, remote);
else
dir = g_object_ref (self->local_heads_dir);
child = g_file_resolve_relative_path (dir, ref_prefix);
if (!ot_gfile_query_info_allow_noent (child, OSTREE_GIO_FAST_QUERYINFO, 0,
&info, cancellable, error))
goto out;
if (info)
{
if (g_file_info_get_file_type (info) == G_FILE_TYPE_DIRECTORY)
prefix_path = glnx_strjoina ("refs/remotes/", remote, "/");
path = glnx_strjoina (prefix_path, ref_prefix);
}
else
{
prefix_path = "refs/heads/";
path = glnx_strjoina (prefix_path, ref_prefix);
}
if (fstatat (self->repo_dir_fd, path, &stbuf, 0) < 0)
{
if (errno != ENOENT)
{
if (!enumerate_refs_recurse (self, remote, child, child,
ret_all_refs,
cancellable, error))
glnx_set_error_from_errno (error);
goto out;
}
}
else
{
if (S_ISDIR (stbuf.st_mode))
{
glnx_fd_close int base_fd = -1;
g_autoptr(GString) base_path = g_string_new ("");
if (!glnx_opendirat (self->repo_dir_fd, path, TRUE, &base_fd, error))
goto out;
if (!enumerate_refs_recurse (self, remote, base_fd, base_path,
base_fd, ".",
ret_all_refs, cancellable, error))
goto out;
}
else
{
if (!add_ref_to_set (remote, dir, child, ret_all_refs,
glnx_fd_close int prefix_dfd = -1;
if (!glnx_opendirat (self->repo_dir_fd, prefix_path, TRUE, &prefix_dfd, error))
goto out;
if (!add_ref_to_set (remote, prefix_dfd, ref_prefix, ret_all_refs,
cancellable, error))
goto out;
}
@@ -541,31 +592,41 @@ ostree_repo_list_refs (OstreeRepo *self,
}
else
{
g_autoptr(GFileEnumerator) remote_enumerator = NULL;
if (!enumerate_refs_recurse (self, NULL, self->local_heads_dir, self->local_heads_dir,
ret_all_refs,
cancellable, error))
g_auto(GLnxDirFdIterator) dfd_iter = { 0, };
g_autoptr(GString) base_path = g_string_new ("");
glnx_fd_close int refs_heads_dfd = -1;
if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &refs_heads_dfd, error))
goto out;
remote_enumerator = g_file_enumerate_children (self->remote_heads_dir, OSTREE_GIO_FAST_QUERYINFO,
0,
cancellable, error);
if (!enumerate_refs_recurse (self, NULL, refs_heads_dfd, base_path,
refs_heads_dfd, ".",
ret_all_refs, cancellable, error))
goto out;
g_string_truncate (base_path, 0);
if (!glnx_dirfd_iterator_init_at (self->repo_dir_fd, "refs/remotes", TRUE, &dfd_iter, error))
goto out;
while (TRUE)
{
GFileInfo *info;
GFile *child;
const char *name;
struct dirent *dent;
glnx_fd_close int remote_dfd = -1;
if (!gs_file_enumerator_iterate (remote_enumerator, &info, &child,
cancellable, error))
if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error))
goto out;
if (!info)
if (!dent)
break;
name = g_file_info_get_name (info);
if (!enumerate_refs_recurse (self, name, child, child,
if (dent->d_type != DT_DIR)
continue;
if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &remote_dfd, error))
goto out;
if (!enumerate_refs_recurse (self, dent->d_name, remote_dfd, base_path,
remote_dfd, ".",
ret_all_refs,
cancellable, error))
goto out;
src/libostree/ostree-repo-static-delta-compilation.c
+37 -8
View File
@@ -59,6 +59,7 @@ typedef struct {
guint n_rollsum;
guint n_bsdiff;
guint n_fallback;
gboolean swap_endian;
} OstreeStaticDeltaBuilder;
typedef enum {
@@ -228,7 +229,7 @@ objtype_checksum_array_new (GPtrArray *objects)
GVariant *serialized_key = objects->pdata[i];
OstreeObjectType objtype;
const char *checksum;
guint8 csum[32];
guint8 csum[OSTREE_SHA256_DIGEST_LEN];
guint8 objtype_v;
ostree_object_name_deserialize (serialized_key, &checksum, &objtype);
@@ -678,7 +679,7 @@ process_one_rollsum (OstreeRepo *repo,
{ gsize mode_offset, xattr_offset, from_csum_offset;
gboolean reading_payload = TRUE;
guchar source_csum[32];
guchar source_csum[OSTREE_SHA256_DIGEST_LEN];
guint i;
write_content_mode_xattrs (repo, current_part, content_finfo, content_xattrs,
@@ -799,7 +800,7 @@ process_one_bsdiff (OstreeRepo *repo,
g_ptr_array_add (current_part->objects, ostree_object_name_serialize (to_checksum, OSTREE_OBJECT_TYPE_FILE));
{ gsize mode_offset, xattr_offset;
guchar source_csum[32];
guchar source_csum[OSTREE_SHA256_DIGEST_LEN];
write_content_mode_xattrs (repo, current_part, content_finfo, content_xattrs,
&mode_offset, &xattr_offset);
@@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo *self,
g_variant_new ("(y@aytt)",
objtype,
ostree_checksum_to_bytes_v (checksum),
compressed_size, uncompressed_size));
maybe_swap_endian_u64 (builder->swap_endian, compressed_size),
maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size)));
}
ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder));
@@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo *self,
* - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE.
* - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE.
* - verbose: b: Print diagnostic messages. Default FALSE.
* - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN)
* - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves to repository.
*/
gboolean
@@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
g_autoptr(GVariant) fallback_headers = NULL;
g_autoptr(GVariant) detached = NULL;
gboolean inline_parts;
guint endianness = G_BYTE_ORDER;
g_autoptr(GFile) tmp_dir = NULL;
builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref);
builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
@@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
max_chunk_size = 32;
builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000;
(void) g_variant_lookup (params, "endianness", "u", &endianness);
g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE);
builder.swap_endian = endianness != G_BYTE_ORDER;
{ gboolean use_bsdiff;
if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff))
use_bsdiff = TRUE;
@@ -1306,6 +1315,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
cancellable, error))
goto out;
/* NOTE: Add user-supplied metadata first. This is used by at least
* xdg-app as a way to provide MIME content sniffing, since the
* metadata appears first in the file.
*/
g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}"));
if (metadata != NULL)
{
@@ -1320,6 +1333,22 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
}
}
{ guint8 endianness_char;
switch (endianness)
{
case G_LITTLE_ENDIAN:
endianness_char = 'l';
break;
case G_BIG_ENDIAN:
endianness_char = 'B';
break;
default:
g_assert_not_reached ();
}
g_variant_builder_add (&metadata_builder, "{sv}", "ostree.endianness", g_variant_new_byte (endianness_char));
}
if (opt_filename)
{
g_autoptr(GFile) f = g_file_new_for_path (opt_filename);
@@ -1411,13 +1440,13 @@ ostree_repo_static_delta_generate (OstreeRepo *self,
cancellable, error))
goto out;
checksum_bytes = g_bytes_new (part_checksum, 32);
checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN);
objtype_checksum_array = objtype_checksum_array_new (part_builder->objects);
delta_part_header = g_variant_new ("(u@aytt@ay)",
OSTREE_DELTAPART_VERSION,
maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION),
ot_gvariant_new_ay_bytes (checksum_bytes),
(guint64) g_variant_get_size (delta_part),
part_builder->uncompressed_size,
maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)),
maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size),
ot_gvariant_new_ay_bytes (objtype_checksum_array));
g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header));
src/libostree/ostree-repo-static-delta-core.c
+539 -36
View File
@@ -20,8 +20,14 @@
#include "config.h"
#include <gio/gunixinputstream.h>
#include <gio/gunixoutputstream.h>
#include <gio/gfiledescriptorbased.h>
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
#include "ostree-lzma-decompressor.h"
#include "ostree-cmdprivate.h"
#include "ostree-checksum-input-stream.h"
#include "ostree-repo-static-delta-private.h"
#include "otutil.h"
@@ -131,7 +137,7 @@ ostree_repo_list_static_delta_names (OstreeRepo *self,
g_autofree char *buf = g_strconcat (name1, name2, NULL);
GString *out = g_string_new ("");
char checksum[65];
guchar csum[32];
guchar csum[OSTREE_SHA256_DIGEST_LEN];
const char *dash = strchr (buf, '-');
ostree_checksum_b64_inplace_to_bytes (buf, csum);
@@ -225,31 +231,49 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
{
gboolean ret = FALSE;
guint i, n;
g_autoptr(GFile) meta_file = NULL;
g_autoptr(GFile) dir = NULL;
const char *dir_or_file_path = NULL;
glnx_fd_close int meta_fd = -1;
glnx_fd_close int dfd = -1;
g_autoptr(GVariant) meta = NULL;
g_autoptr(GVariant) headers = NULL;
g_autoptr(GVariant) metadata = NULL;
g_autoptr(GVariant) fallback = NULL;
g_autofree char *to_checksum = NULL;
g_autofree char *from_checksum = NULL;
GFileType file_type;
g_autofree char *basename = NULL;
dir_or_file_path = gs_file_get_path_cached (dir_or_file);
file_type = g_file_query_file_type (dir_or_file, 0, cancellable);
if (file_type == G_FILE_TYPE_DIRECTORY)
/* First, try opening it as a directory */
dfd = glnx_opendirat_with_errno (AT_FDCWD, dir_or_file_path, TRUE);
if (dfd < 0)
{
dir = g_object_ref (dir_or_file);
meta_file = g_file_get_child (dir, "superblock");
if (errno != ENOTDIR)
{
glnx_set_error_from_errno (error);
goto out;
}
else
{
g_autofree char *dir = dirname (g_strdup (dir_or_file_path));
basename = g_path_get_basename (dir_or_file_path);
if (!glnx_opendirat (AT_FDCWD, dir, TRUE, &dfd, error))
goto out;
}
}
else
{
meta_file = g_object_ref (dir_or_file);
dir = g_file_get_parent (meta_file);
}
basename = g_strdup ("superblock");
if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
FALSE, &meta, error))
meta_fd = openat (dfd, basename, O_RDONLY | O_CLOEXEC);
if (meta_fd < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
if (!ot_util_variant_map_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT),
FALSE, &meta, error))
goto out;
/* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */
@@ -329,14 +353,18 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
guint64 size;
guint64 usize;
const guchar *csum;
char checksum[65];
gboolean have_all;
g_autoptr(GInputStream) part_in = NULL;
g_autoptr(GBytes) delta_data = NULL;
g_autoptr(GVariant) part_data = NULL;
g_autoptr(GVariant) inline_part_data = NULL;
g_autoptr(GVariant) header = NULL;
g_autoptr(GVariant) csum_v = NULL;
g_autoptr(GVariant) objects = NULL;
g_autoptr(GBytes) bytes = NULL;
g_autoptr(GVariant) part = NULL;
g_autofree char *deltapart_path = NULL;
OstreeStaticDeltaOpenFlags delta_open_flags =
skip_validation ? OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM : 0;
header = g_variant_get_child_value (headers, i);
g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects);
@@ -361,41 +389,57 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
csum = ostree_checksum_bytes_peek_validate (csum_v, error);
if (!csum)
goto out;
ostree_checksum_inplace_from_bytes (csum, checksum);
deltapart_path =
_ostree_get_relative_static_delta_part_path (from_checksum, to_checksum, i);
part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)"));
if (part_data)
inline_part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)"));
if (inline_part_data)
{
bytes = g_variant_get_data_as_bytes (part_data);
g_autoptr(GBytes) inline_part_bytes = g_variant_get_data_as_bytes (inline_part_data);
part_in = g_memory_input_stream_new_from_bytes (inline_part_bytes);
/* For inline parts, we don't checksum, because it's
* included with the metadata, so we're not trying to
* protect against MITM or such. Non-security related
* checksums should be done at the underlying storage layer.
*/
delta_open_flags |= OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM;
if (!_ostree_static_delta_part_open (part_in, inline_part_bytes,
delta_open_flags,
NULL,
&part,
cancellable, error))
goto out;
}
else
{
g_autoptr(GFile) part_path = ot_gfile_resolve_path_printf (dir, "%u", i);
GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error);
if (!mfile)
goto out;
g_autofree char *relpath = g_strdup_printf ("%u", i); /* TODO avoid malloc here */
glnx_fd_close int part_fd = openat (dfd, relpath, O_RDONLY | O_CLOEXEC);
if (part_fd < 0)
{
glnx_set_error_from_errno (error);
g_prefix_error (error, "Opening deltapart '%s': ", deltapart_path);
goto out;
}
bytes = g_mapped_file_get_bytes (mfile);
g_mapped_file_unref (mfile);
}
part_in = g_unix_input_stream_new (part_fd, FALSE);
if (!skip_validation)
{
g_autoptr(GInputStream) in = g_memory_input_stream_new_from_bytes (bytes);
g_autofree char *expected_checksum = ostree_checksum_from_bytes (csum);
if (!_ostree_static_delta_part_validate (self, in, i,
expected_checksum,
cancellable, error))
if (!_ostree_static_delta_part_open (part_in, NULL,
delta_open_flags,
checksum,
&part,
cancellable, error))
goto out;
}
if (!_ostree_static_delta_part_execute (self, objects, bytes, skip_validation,
if (!_ostree_static_delta_part_execute (self, objects, part, skip_validation,
FALSE, NULL,
cancellable, error))
{
g_prefix_error (error, "executing delta part %i: ", i);
g_prefix_error (error, "Executing delta part %i: ", i);
goto out;
}
}
@@ -404,3 +448,462 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self,
out:
return ret;
}
gboolean
_ostree_static_delta_part_open (GInputStream *part_in,
GBytes *inline_part_bytes,
OstreeStaticDeltaOpenFlags flags,
const char *expected_checksum,
GVariant **out_part,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
const gboolean trusted = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED) > 0;
const gboolean skip_checksum = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM) > 0;
gsize bytes_read;
guint8 comptype;
g_autoptr(GChecksum) checksum = NULL;
g_autoptr(GInputStream) checksum_in = NULL;
g_autoptr(GVariant) ret_part = NULL;
GInputStream *source_in;
/* We either take a fd or a GBytes reference */
g_return_val_if_fail (G_IS_FILE_DESCRIPTOR_BASED (part_in) || inline_part_bytes != NULL, FALSE);
g_return_val_if_fail (skip_checksum || expected_checksum != NULL, FALSE);
if (!skip_checksum)
{
checksum = g_checksum_new (G_CHECKSUM_SHA256);
checksum_in = (GInputStream*)ostree_checksum_input_stream_new (part_in, checksum);
source_in = checksum_in;
}
else
{
source_in = part_in;
}
{ guint8 buf[1];
/* First byte is compression type */
if (!g_input_stream_read_all (source_in, buf, sizeof(buf), &bytes_read,
cancellable, error))
{
g_prefix_error (error, "Reading initial compression flag byte: ");
goto out;
}
comptype = buf[0];
}
switch (comptype)
{
case 0:
if (!inline_part_bytes)
{
int part_fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)part_in);
/* No compression, no checksums - a fast path */
if (!ot_util_variant_map_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
trusted, &ret_part, error))
goto out;
}
else
{
g_autoptr(GBytes) content_bytes = g_bytes_new_from_bytes (inline_part_bytes, 1,
g_bytes_get_size (inline_part_bytes) - 1);
ret_part = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
content_bytes, trusted);
}
if (!skip_checksum)
g_checksum_update (checksum, g_variant_get_data (ret_part),
g_variant_get_size (ret_part));
break;
case 'x':
{
g_autofree char *tmppath = g_strdup ("/var/tmp/ostree-delta-XXXXXX");
g_autoptr(GConverter) decomp = (GConverter*) _ostree_lzma_decompressor_new ();
g_autoptr(GInputStream) convin = g_converter_input_stream_new (source_in, decomp);
g_autoptr(GOutputStream) unpacked_out = NULL;
glnx_fd_close int unpacked_fd = -1;
gssize n_bytes_written;
unpacked_fd = g_mkstemp_full (tmppath, O_RDWR | O_CLOEXEC, 0640);
if (unpacked_fd < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
/* Now make it autocleanup on process exit - in the future, we
* should consider caching unpacked deltas as well.
*/
if (unlink (tmppath) < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
unpacked_out = g_unix_output_stream_new (unpacked_fd, FALSE);
n_bytes_written = g_output_stream_splice (unpacked_out, convin,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error);
if (n_bytes_written < 0)
goto out;
if (!ot_util_variant_map_fd (unpacked_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
trusted, &ret_part, error))
goto out;
}
break;
default:
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid compression type '%u'", comptype);
goto out;
}
if (checksum)
{
const char *actual_checksum = g_checksum_get_string (checksum);
g_assert (expected_checksum != NULL);
if (strcmp (actual_checksum, expected_checksum) != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Checksum mismatch in static delta part; expected=%s actual=%s",
expected_checksum, actual_checksum);
goto out;
}
}
ret = TRUE;
*out_part = g_steal_pointer (&ret_part);
out:
return ret;
}
/*
* Displaying static delta parts
*/
static gboolean
show_one_part (OstreeRepo *self,
gboolean swap_endian,
const char *from,
const char *to,
GVariant *meta_entries,
guint i,
guint64 *total_size_ref,
guint64 *total_usize_ref,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint32 version;
guint64 size, usize;
g_autoptr(GVariant) objects = NULL;
g_autoptr(GInputStream) part_in = NULL;
g_autoptr(GVariant) part = NULL;
g_autofree char *part_path = _ostree_get_relative_static_delta_part_path (from, to, i);
gint part_fd = -1;
g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects);
size = maybe_swap_endian_u64 (swap_endian, size);
usize = maybe_swap_endian_u64 (swap_endian, usize);
*total_size_ref += size;
*total_usize_ref += usize;
g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n",
i, (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN), size, usize);
part_fd = openat (self->repo_dir_fd, part_path, O_RDONLY | O_CLOEXEC);
if (part_fd < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
part_in = g_unix_input_stream_new (part_fd, FALSE);
if (!_ostree_static_delta_part_open (part_in, NULL,
OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM,
NULL,
&part,
cancellable, error))
goto out;
{ g_autoptr(GVariant) modes = NULL;
g_autoptr(GVariant) xattrs = NULL;
g_autoptr(GVariant) blob = NULL;
g_autoptr(GVariant) ops = NULL;
OstreeDeltaExecuteStats stats = { { 0, }, };
g_variant_get (part, "(@a(uuu)@aa(ayay)@ay@ay)",
&modes, &xattrs, &blob, &ops);
g_print ("PartPayload%u: nmodes=%" G_GUINT64_FORMAT
" nxattrs=%" G_GUINT64_FORMAT
" blobsize=%" G_GUINT64_FORMAT
" opsize=%" G_GUINT64_FORMAT
"\n",
i,
(guint64)g_variant_n_children (modes),
(guint64)g_variant_n_children (xattrs),
(guint64)g_variant_n_children (blob),
(guint64)g_variant_n_children (ops));
if (!_ostree_static_delta_part_execute (self, objects,
part, TRUE, TRUE,
&stats, cancellable, error))
goto out;
{ const guint *n_ops = stats.n_ops_executed;
g_print ("PartPayloadOps%u: openspliceclose=%u open=%u write=%u setread=%u "
"unsetread=%u close=%u bspatch=%u\n",
i, n_ops[0], n_ops[1], n_ops[2], n_ops[3], n_ops[4], n_ops[5], n_ops[6]);
}
}
ret = TRUE;
out:
return ret;
}
OstreeDeltaEndianness
_ostree_delta_get_endianness (GVariant *superblock,
gboolean *out_was_heuristic)
{
guint8 endianness_char;
g_autoptr(GVariant) delta_meta = NULL;
g_autoptr(GVariantDict) delta_metadict = NULL;
guint64 total_size = 0;
guint64 total_usize = 0;
guint total_objects = 0;
delta_meta = g_variant_get_child_value (superblock, 0);
delta_metadict = g_variant_dict_new (delta_meta);
if (out_was_heuristic)
*out_was_heuristic = FALSE;
if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char))
{
switch (endianness_char)
{
case 'l':
return OSTREE_DELTA_ENDIAN_LITTLE;
case 'B':
return OSTREE_DELTA_ENDIAN_BIG;
default:
return OSTREE_DELTA_ENDIAN_INVALID;
}
}
if (out_was_heuristic)
*out_was_heuristic = TRUE;
{ g_autoptr(GVariant) meta_entries = NULL;
guint n_parts;
guint i;
gboolean is_byteswapped = FALSE;
g_variant_get_child (superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries);
n_parts = g_variant_n_children (meta_entries);
for (i = 0; i < n_parts; i++)
{
g_autoptr(GVariant) objects = NULL;
guint64 size, usize;
guint n_objects;
g_variant_get_child (meta_entries, i, "(u@aytt@ay)", NULL, NULL, &size, &usize, &objects);
n_objects = (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN);
total_objects += n_objects;
total_size += size;
total_usize += usize;
if (size > usize)
{
double ratio = ((double)size)/((double)usize);
/* This should really never happen where compressing things makes it more than 50% bigger.
*/
if (ratio > 1.2)
{
is_byteswapped = TRUE;
break;
}
}
}
if (!is_byteswapped)
{
/* If the average object size is greater than 4GiB, let's assume
* we're dealing with opposite endianness. I'm fairly confident
* no one is going to be shipping peta- or exa- byte size ostree
* deltas, period. Past the gigabyte scale you really want
* bittorrent or something.
*/
if ((total_size / total_objects) > G_MAXUINT32)
{
is_byteswapped = TRUE;
}
}
if (is_byteswapped)
{
switch (G_BYTE_ORDER)
{
case G_BIG_ENDIAN:
return OSTREE_DELTA_ENDIAN_LITTLE;
case G_LITTLE_ENDIAN:
return OSTREE_DELTA_ENDIAN_BIG;
default:
g_assert_not_reached ();
}
}
return G_BYTE_ORDER;
}
}
gboolean
_ostree_delta_needs_byteswap (GVariant *superblock)
{
switch (_ostree_delta_get_endianness (superblock, NULL))
{
case OSTREE_DELTA_ENDIAN_BIG:
return G_BYTE_ORDER == G_LITTLE_ENDIAN;
case OSTREE_DELTA_ENDIAN_LITTLE:
return G_BYTE_ORDER == G_BIG_ENDIAN;
default:
return FALSE;
}
}
gboolean
_ostree_repo_static_delta_dump (OstreeRepo *self,
const char *delta_id,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autofree char *from = NULL;
g_autofree char *to = NULL;
g_autofree char *superblock_path = NULL;
glnx_fd_close int superblock_fd = -1;
g_autoptr(GVariant) delta_superblock = NULL;
guint64 total_size = 0, total_usize = 0;
guint64 total_fallback_size = 0, total_fallback_usize = 0;
guint i;
OstreeDeltaEndianness endianness;
gboolean swap_endian = FALSE;
_ostree_parse_delta_name (delta_id, &from, &to);
superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path,
(GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT,
TRUE, &delta_superblock, error))
goto out;
g_print ("%s\n", g_variant_print (delta_superblock, 1));
g_print ("Delta: %s\n", delta_id);
{ const char *endianness_description;
gboolean was_heuristic;
endianness = _ostree_delta_get_endianness (delta_superblock, &was_heuristic);
switch (endianness)
{
case OSTREE_DELTA_ENDIAN_BIG:
if (was_heuristic)
endianness_description = "big (heuristic)";
else
endianness_description = "big";
if (G_BYTE_ORDER == G_LITTLE_ENDIAN)
swap_endian = TRUE;
break;
case OSTREE_DELTA_ENDIAN_LITTLE:
if (was_heuristic)
endianness_description = "little (heuristic)";
else
endianness_description = "little";
if (G_BYTE_ORDER == G_BIG_ENDIAN)
swap_endian = TRUE;
break;
case OSTREE_DELTA_ENDIAN_INVALID:
endianness_description = "invalid";
break;
default:
g_assert_not_reached ();
}
g_print ("Endianness: %s\n", endianness_description);
}
{ guint64 ts;
g_variant_get_child (delta_superblock, 1, "t", &ts);
g_print ("Timestamp: %" G_GUINT64_FORMAT "\n", GUINT64_FROM_BE (ts));
}
{ g_autoptr(GVariant) recurse = NULL;
g_variant_get_child (delta_superblock, 5, "@ay", &recurse);
g_print ("Number of parents: %u\n", (guint)(g_variant_get_size (recurse) / (OSTREE_SHA256_DIGEST_LEN * 2)));
}
{ g_autoptr(GVariant) fallback = NULL;
guint n_fallback;
g_variant_get_child (delta_superblock, 7, "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT, &fallback);
n_fallback = g_variant_n_children (fallback);
g_print ("Number of fallback entries: %u\n", n_fallback);
for (i = 0; i < n_fallback; i++)
{
guint64 size, usize;
g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize);
size = maybe_swap_endian_u64 (swap_endian, size);
usize = maybe_swap_endian_u64 (swap_endian, usize);
total_fallback_size += size;
total_fallback_usize += usize;
}
{ g_autofree char *sizestr = g_format_size (total_fallback_size);
g_autofree char *usizestr = g_format_size (total_fallback_usize);
g_print ("Total Fallback Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_size, sizestr);
g_print ("Total Fallback Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_usize, usizestr);
}
}
{ g_autoptr(GVariant) meta_entries = NULL;
guint n_parts;
g_variant_get_child (delta_superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries);
n_parts = g_variant_n_children (meta_entries);
g_print ("Number of parts: %u\n", n_parts);
for (i = 0; i < n_parts; i++)
{
if (!show_one_part (self, swap_endian, from, to, meta_entries, i,
&total_size, &total_usize,
cancellable, error))
goto out;
}
}
{ g_autofree char *sizestr = g_format_size (total_size);
g_autofree char *usizestr = g_format_size (total_usize);
g_print ("Total Part Size: %" G_GUINT64_FORMAT " (%s)\n", total_size, sizestr);
g_print ("Total Part Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_usize, usizestr);
}
{ guint64 overall_size = total_size + total_fallback_size;
guint64 overall_usize = total_usize + total_fallback_usize;
g_autofree char *sizestr = g_format_size (overall_size);
g_autofree char *usizestr = g_format_size (overall_usize);
g_print ("Total Size: %" G_GUINT64_FORMAT " (%s)\n", overall_size, sizestr);
g_print ("Total Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", overall_usize, usizestr);
}
ret = TRUE;
out:
return ret;
}
src/libostree/ostree-repo-static-delta-private.h
+90 -38
View File
@@ -46,10 +46,10 @@ G_BEGIN_DECLS
/**
* OSTREE_STATIC_DELTA_META_ENTRY_FORMAT:
*
* u: version
* u: version (non-canonical endian)
* ay checksum
* guint64 size: Total size of delta (sum of parts)
* guint64 usize: Uncompressed size of resulting objects on disk
* guint64 size: Total size of delta (sum of parts) (non-canonical endian)
* guint64 usize: Uncompressed size of resulting objects on disk (non-canonical endian)
* ARRAY[(guint8 objtype, csum object)]
*
* The checksum is of the delta payload, and each entry in the array
@@ -64,8 +64,8 @@ G_BEGIN_DECLS
*
* y: objtype
* ay: checksum
* t: compressed size
* t: uncompressed size
* t: compressed size (non-canonical endian)
* t: uncompressed size (non-canonical endian)
*
* Object to fetch invididually; includes compressed/uncompressed size.
*/
@@ -79,7 +79,7 @@ G_BEGIN_DECLS
*
* delta-descriptor:
* metadata: a{sv}
* t: timestamp
* t: timestamp (big endian)
* from: ay checksum
* to: ay checksum
* commit: new commit object
@@ -103,38 +103,11 @@ G_BEGIN_DECLS
*/
#define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" OSTREE_COMMIT_GVARIANT_STRING "aya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")"
gboolean _ostree_static_delta_part_validate (OstreeRepo *repo,
GInputStream *in,
guint part_offset,
const char *expected_checksum,
GCancellable *cancellable,
GError **error);
gboolean _ostree_static_delta_part_execute (OstreeRepo *repo,
GVariant *header,
GBytes *partdata,
gboolean trusted,
GCancellable *cancellable,
GError **error);
gboolean _ostree_static_delta_part_execute_raw (OstreeRepo *repo,
GVariant *header,
GVariant *part,
gboolean trusted,
GCancellable *cancellable,
GError **error);
void _ostree_static_delta_part_execute_async (OstreeRepo *repo,
GVariant *header,
GBytes *partdata,
gboolean trusted,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data);
gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo,
GAsyncResult *result,
GError **error);
typedef enum {
OSTREE_STATIC_DELTA_OPEN_FLAGS_NONE = 0,
OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM = (1 << 0),
OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED = (1 << 1)
} OstreeStaticDeltaOpenFlags;
typedef enum {
OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE = 'S',
@@ -145,6 +118,46 @@ typedef enum {
OSTREE_STATIC_DELTA_OP_CLOSE = 'c',
OSTREE_STATIC_DELTA_OP_BSPATCH = 'B'
} OstreeStaticDeltaOpCode;
#define OSTREE_STATIC_DELTA_N_OPS 7
gboolean
_ostree_static_delta_part_open (GInputStream *part_in,
GBytes *inline_part_bytes,
OstreeStaticDeltaOpenFlags flags,
const char *expected_checksum,
GVariant **out_part,
GCancellable *cancellable,
GError **error);
gboolean _ostree_static_delta_dump (OstreeRepo *repo,
const char *delta_id,
GCancellable *cancellable,
GError **error);
typedef struct {
guint n_ops_executed[OSTREE_STATIC_DELTA_N_OPS];
} OstreeDeltaExecuteStats;
gboolean _ostree_static_delta_part_execute (OstreeRepo *repo,
GVariant *header,
GVariant *part_payload,
gboolean trusted,
gboolean stats_only,
OstreeDeltaExecuteStats *stats,
GCancellable *cancellable,
GError **error);
void _ostree_static_delta_part_execute_async (OstreeRepo *repo,
GVariant *header,
GVariant *part_payload,
gboolean trusted,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data);
gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo,
GAsyncResult *result,
GError **error);
gboolean
_ostree_static_delta_parse_checksum_array (GVariant *array,
@@ -177,4 +190,43 @@ _ostree_delta_compute_similar_objects (OstreeRepo *repo,
GCancellable *cancellable,
GError **error);
gboolean
_ostree_repo_static_delta_dump (OstreeRepo *repo,
const char *delta_id,
GCancellable *cancellable,
GError **error);
/* Used for static deltas which due to a historical mistake are
* inconsistent endian.
*
* https://bugzilla.gnome.org/show_bug.cgi?id=762515
*/
static inline guint32
maybe_swap_endian_u32 (gboolean swap,
guint32 v)
{
if (!swap)
return v;
return GUINT32_SWAP_LE_BE (v);
}
static inline guint64
maybe_swap_endian_u64 (gboolean swap,
guint64 v)
{
if (!swap)
return v;
return GUINT64_SWAP_LE_BE (v);
}
typedef enum {
OSTREE_DELTA_ENDIAN_BIG,
OSTREE_DELTA_ENDIAN_LITTLE,
OSTREE_DELTA_ENDIAN_INVALID
} OstreeDeltaEndianness;
OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock, gboolean *out_was_heuristic);
gboolean _ostree_delta_needs_byteswap (GVariant *superblock);
G_END_DECLS
src/libostree/ostree-repo-static-delta-processing.c
+87 -128
View File
@@ -40,6 +40,7 @@ G_STATIC_ASSERT (sizeof (guint) >= sizeof (guint32));
typedef struct {
gboolean trusted;
gboolean stats_only;
OstreeRepo *repo;
guint checksum_index;
const guint8 *checksums;
@@ -149,43 +150,39 @@ open_output_target (StaticDeltaExecutionState *state,
return ret;
}
gboolean
_ostree_static_delta_part_validate (OstreeRepo *repo,
GInputStream *in,
guint part_offset,
const char *expected_checksum,
GCancellable *cancellable,
GError **error)
static guint
delta_opcode_index (OstreeStaticDeltaOpCode op)
{
gboolean ret = FALSE;
g_autofree guchar *actual_checksum_bytes = NULL;
g_autofree char *actual_checksum = NULL;
if (!ot_gio_checksum_stream (in, &actual_checksum_bytes,
cancellable, error))
goto out;
actual_checksum = ostree_checksum_from_bytes (actual_checksum_bytes);
if (strcmp (actual_checksum, expected_checksum) != 0)
switch (op)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Checksum mismatch in static delta part %u; expected=%s actual=%s",
part_offset, expected_checksum, actual_checksum);
goto out;
case OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE:
return 0;
case OSTREE_STATIC_DELTA_OP_OPEN:
return 1;
case OSTREE_STATIC_DELTA_OP_WRITE:
return 2;
case OSTREE_STATIC_DELTA_OP_SET_READ_SOURCE:
return 3;
case OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE:
return 4;
case OSTREE_STATIC_DELTA_OP_CLOSE:
return 5;
case OSTREE_STATIC_DELTA_OP_BSPATCH:
return 6;
default:
g_assert_not_reached ();
}
ret = TRUE;
out:
return ret;
}
gboolean
_ostree_static_delta_part_execute_raw (OstreeRepo *repo,
GVariant *objects,
GVariant *part,
gboolean trusted,
GCancellable *cancellable,
GError **error)
_ostree_static_delta_part_execute (OstreeRepo *repo,
GVariant *objects,
GVariant *part,
gboolean trusted,
gboolean stats_only,
OstreeDeltaExecuteStats *stats,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint8 *checksums_data;
@@ -201,6 +198,7 @@ _ostree_static_delta_part_execute_raw (OstreeRepo *repo,
state->repo = repo;
state->async_error = error;
state->trusted = trusted;
state->stats_only = stats_only;
if (!_ostree_static_delta_parse_checksum_array (objects,
&checksums_data,
@@ -270,6 +268,8 @@ _ostree_static_delta_part_execute_raw (OstreeRepo *repo,
}
n_executed++;
if (stats)
stats->n_ops_executed[delta_opcode_index(opcode)]++;
}
if (state->caught_error)
@@ -280,99 +280,10 @@ _ostree_static_delta_part_execute_raw (OstreeRepo *repo,
return ret;
}
static gboolean
decompress_all (GConverter *converter,
GBytes *data,
GBytes **out_uncompressed,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GMemoryInputStream) memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes (data);
g_autoptr(GMemoryOutputStream) memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free);
g_autoptr(GInputStream) convin = g_converter_input_stream_new ((GInputStream*)memin, converter);
{
gssize n_bytes_written = g_output_stream_splice ((GOutputStream*)memout, convin,
G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE |
G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET,
cancellable, error);
if (n_bytes_written < 0)
goto out;
}
ret = TRUE;
*out_uncompressed = g_memory_output_stream_steal_as_bytes (memout);
out:
return ret;
}
gboolean
_ostree_static_delta_part_execute (OstreeRepo *repo,
GVariant *header,
GBytes *part_bytes,
gboolean trusted,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gsize partlen;
const guint8*partdata;
g_autoptr(GBytes) part_payload_bytes = NULL;
g_autoptr(GBytes) payload_data = NULL;
g_autoptr(GVariant) payload = NULL;
guint8 comptype;
partdata = g_bytes_get_data (part_bytes, &partlen);
if (partlen < 1)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Corrupted 0 length delta part");
goto out;
}
/* First byte is compression type */
comptype = partdata[0];
/* Then the rest may be compressed or uncompressed */
part_payload_bytes = g_bytes_new_from_bytes (part_bytes, 1, partlen - 1);
switch (comptype)
{
case 0:
/* No compression */
payload_data = g_bytes_ref (part_payload_bytes);
break;
case 'x':
{
g_autoptr(GConverter) decomp =
(GConverter*) _ostree_lzma_decompressor_new ();
if (!decompress_all (decomp, part_payload_bytes, &payload_data,
cancellable, error))
goto out;
}
break;
default:
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid compression type '%u'", comptype);
goto out;
}
payload = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0),
payload_data, FALSE);
if (!_ostree_static_delta_part_execute_raw (repo, header, payload, trusted,
cancellable, error))
goto out;
ret = TRUE;
out:
return ret;
}
typedef struct {
OstreeRepo *repo;
GVariant *header;
GBytes *partdata;
GVariant *part;
GCancellable *cancellable;
GSimpleAsyncResult *result;
gboolean trusted;
@@ -385,7 +296,7 @@ static_delta_part_execute_async_data_free (gpointer user_data)
g_clear_object (&data->repo);
g_variant_unref (data->header);
g_bytes_unref (data->partdata);
g_variant_unref (data->part);
g_clear_object (&data->cancellable);
g_free (data);
}
@@ -401,8 +312,9 @@ static_delta_part_execute_thread (GSimpleAsyncResult *res,
data = g_simple_async_result_get_op_res_gpointer (res);
if (!_ostree_static_delta_part_execute (data->repo,
data->header,
data->partdata,
data->part,
data->trusted,
FALSE, NULL,
cancellable, &error))
g_simple_async_result_take_error (res, error);
}
@@ -410,7 +322,7 @@ static_delta_part_execute_thread (GSimpleAsyncResult *res,
void
_ostree_static_delta_part_execute_async (OstreeRepo *repo,
GVariant *header,
GBytes *partdata,
GVariant *part,
gboolean trusted,
GCancellable *cancellable,
GAsyncReadyCallback callback,
@@ -421,7 +333,7 @@ _ostree_static_delta_part_execute_async (OstreeRepo *repo,
asyncdata = g_new0 (StaticDeltaPartExecuteAsyncData, 1);
asyncdata->repo = g_object_ref (repo);
asyncdata->header = g_variant_ref (header);
asyncdata->partdata = g_bytes_ref (partdata);
asyncdata->part = g_variant_ref (part);
asyncdata->trusted = trusted;
asyncdata->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
@@ -538,6 +450,12 @@ dispatch_bspatch (OstreeRepo *repo,
if (!read_varuint64 (state, &length, error))
goto out;
if (state->stats_only)
{
ret = TRUE;
goto out;
}
if (!state->have_obj)
{
input_mfile = g_mapped_file_new_from_fd (state->read_source_fd, FALSE, error);
@@ -596,6 +514,12 @@ dispatch_open_splice_and_close (OstreeRepo *repo,
goto out;
if (!validate_ofs (state, offset, length, error))
goto out;
if (state->stats_only)
{
ret = TRUE;
goto out;
}
metadata = g_variant_new_from_data (ostree_metadata_variant_type (state->output_objtype),
state->payload_data + offset, length, TRUE, NULL, NULL);
@@ -639,6 +563,12 @@ dispatch_open_splice_and_close (OstreeRepo *repo,
if (!validate_ofs (state, content_offset, state->content_size, error))
goto out;
if (state->stats_only)
{
ret = TRUE;
goto out;
}
/* Fast path for regular files to bare repositories */
if (S_ISREG (state->mode) &&
(repo->mode == OSTREE_REPO_MODE_BARE ||
@@ -730,6 +660,8 @@ dispatch_open_splice_and_close (OstreeRepo *repo,
ret = TRUE;
out:
if (state->stats_only)
(void) dispatch_close (repo, state, cancellable, NULL);
if (!ret)
g_prefix_error (error, "opcode open-splice-and-close: ");
return ret;
@@ -745,8 +677,11 @@ dispatch_open (OstreeRepo *repo,
g_assert (state->output_target == NULL);
/* FIXME - lift this restriction */
g_assert (repo->mode == OSTREE_REPO_MODE_BARE ||
repo->mode == OSTREE_REPO_MODE_BARE_USER);
if (!state->stats_only)
{
g_assert (repo->mode == OSTREE_REPO_MODE_BARE ||
repo->mode == OSTREE_REPO_MODE_BARE_USER);
}
if (!open_output_target (state, cancellable, error))
goto out;
@@ -757,6 +692,12 @@ dispatch_open (OstreeRepo *repo,
if (!read_varuint64 (state, &state->content_size, error))
goto out;
if (state->stats_only)
{
ret = TRUE;
goto out;
}
if (state->trusted)
{
if (!_ostree_repo_open_trusted_content_bare (repo, state->checksum,
@@ -801,6 +742,12 @@ dispatch_write (OstreeRepo *repo,
if (!read_varuint64 (state, &content_offset, error))
goto out;
if (state->stats_only)
{
ret = TRUE;
goto out;
}
if (!state->have_obj)
{
if (state->read_source_fd != -1)
@@ -881,6 +828,12 @@ dispatch_set_read_source (OstreeRepo *repo,
if (!validate_ofs (state, source_offset, 32, error))
goto out;
if (state->stats_only)
{
ret = TRUE;
goto out;
}
g_free (state->read_source_object);
state->read_source_object = ostree_checksum_from_bytes (state->payload_data + source_offset);
@@ -903,6 +856,12 @@ dispatch_unset_read_source (OstreeRepo *repo,
{
gboolean ret = FALSE;
if (state->stats_only)
{
ret = TRUE;
goto out;
}
if (state->read_source_fd)
{
(void) close (state->read_source_fd);
@@ -912,7 +871,7 @@ dispatch_unset_read_source (OstreeRepo *repo,
g_clear_pointer (&state->read_source_object, g_free);
ret = TRUE;
/* out: */
out:
if (!ret)
g_prefix_error (error, "opcode unset-read-source: ");
return ret;
src/libostree/ostree-repo.c
+16 -6
View File
@@ -518,8 +518,6 @@ ostree_repo_finalize (GObject *object)
g_clear_object (&self->tmp_dir);
if (self->tmp_dir_fd)
(void) close (self->tmp_dir_fd);
g_clear_object (&self->local_heads_dir);
g_clear_object (&self->remote_heads_dir);
g_clear_object (&self->objects_dir);
if (self->objects_dir_fd != -1)
(void) close (self->objects_dir_fd);
@@ -605,8 +603,6 @@ ostree_repo_constructed (GObject *object)
g_assert (self->repodir != NULL);
self->tmp_dir = g_file_resolve_relative_path (self->repodir, "tmp");
self->local_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/heads");
self->remote_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/remotes");
self->objects_dir = g_file_get_child (self->repodir, "objects");
self->deltas_dir = g_file_get_child (self->repodir, "deltas");
@@ -2384,6 +2380,19 @@ ostree_repo_set_disable_fsync (OstreeRepo *self,
self->disable_fsync = disable_fsync;
}
/**
* ostree_repo_get_disable_fsync:
* @self: An #OstreeRepo
*
* For more information see ostree_repo_set_disable_fsync().
*
* Returns: Whether or not fsync() is enabled for this repo.
*/
gboolean
ostree_repo_get_disable_fsync (OstreeRepo *self)
{
return self->disable_fsync;
}
/* Replace the contents of a file, honoring the repository's fsync
* policy.
@@ -3775,6 +3784,7 @@ ostree_repo_pull_one_dir (OstreeRepo *self,
* * flags (i): An instance of #OstreeRepoPullFlags
* * refs: (as): Array of string refs
* * depth: (i): How far in the history to traverse; default is 0, -1 means infinite
* * override-commit-ids: (as): Array of specific commit IDs to fetch for refs
*/
gboolean
ostree_repo_pull_with_options (OstreeRepo *self,
@@ -4594,8 +4604,8 @@ ostree_repo_regenerate_summary (OstreeRepo *self,
gs_free char *from = NULL;
gs_free char *to = NULL;
gs_free guchar *csum = NULL;
gs_free char *superblock;
gs_fd_close int superblock_file_fd;
gs_free char *superblock = NULL;
gs_fd_close int superblock_file_fd = -1;
g_autoptr(GInputStream) in_stream = NULL;
_ostree_parse_delta_name (delta_names->pdata[i], &from, &to);
src/libostree/ostree-repo.h
+53 -1
View File
@@ -56,6 +56,8 @@ gboolean ostree_repo_open (OstreeRepo *self,
void ostree_repo_set_disable_fsync (OstreeRepo *self,
gboolean disable_fsync);
gboolean ostree_repo_get_disable_fsync (OstreeRepo *self);
gboolean ostree_repo_is_system (OstreeRepo *repo);
gboolean ostree_repo_is_writable (OstreeRepo *self,
@@ -439,6 +441,7 @@ gboolean ostree_repo_write_dfd_to_mtree (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *self,
GFile *archive,
OstreeMutableTree *mtree,
@@ -447,6 +450,53 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *
GCancellable *cancellable,
GError **error);
/**
* OstreeRepoImportArchiveOptions:
*
* An extensible options structure controlling archive import. Ensure that
* you have entirely zeroed the structure, then set just the desired
* options. This is used by ostree_repo_import_archive_to_mtree().
*/
typedef struct {
guint ignore_unsupported_content : 1;
guint autocreate_parents : 1;
guint reserved : 30;
guint unused_uint[8];
gpointer unused_ptrs[8];
} OstreeRepoImportArchiveOptions;
gboolean ostree_repo_import_archive_to_mtree (OstreeRepo *self,
OstreeRepoImportArchiveOptions *opts,
void *archive, /* Really struct archive * */
OstreeMutableTree *mtree,
OstreeRepoCommitModifier *modifier,
GCancellable *cancellable,
GError **error);
/**
* OstreeRepoExportArchiveOptions:
*
* An extensible options structure controlling archive creation. Ensure that
* you have entirely zeroed the structure, then set just the desired
* options. This is used by ostree_repo_export_tree_to_archive().
*/
typedef struct {
guint disable_xattrs : 1;
guint reserved : 31;
guint64 timestamp_secs;
guint unused_uint[8];
gpointer unused_ptrs[8];
} OstreeRepoExportArchiveOptions;
gboolean ostree_repo_export_tree_to_archive (OstreeRepo *self,
OstreeRepoExportArchiveOptions *opts,
OstreeRepoFile *root,
void *archive, /* Really struct archive * */
GCancellable *cancellable,
GError **error);
gboolean ostree_repo_write_mtree (OstreeRepo *self,
OstreeMutableTree *mtree,
GFile **out_file,
@@ -530,7 +580,9 @@ typedef struct {
OstreeRepoCheckoutOverwriteMode overwrite_mode;
guint enable_uncompressed_cache : 1;
guint unused : 31;
guint disable_fsync : 1;
guint process_whiteouts : 1;
guint reserved : 29;
const char *subpath;
src/libostree/ostree-sysroot-deploy.c
+11 -3
View File
@@ -528,6 +528,11 @@ checkout_deployment_tree (OstreeSysroot *sysroot,
glnx_fd_close int osdeploy_dfd = -1;
int ret_fd;
/* We end up using syncfs for the entire filesystem, so turn off
* OstreeRepo level fsync.
*/
checkout_opts.disable_fsync = TRUE;
osdeploy_path = g_strconcat ("ostree/deploy/", ostree_deployment_get_osname (deployment), "/deploy", NULL);
checkout_target_name = g_strdup_printf ("%s.%d", csum, ostree_deployment_get_deployserial (deployment));
@@ -2030,9 +2035,12 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self,
cancellable, error))
goto out;
if (!ostree_sysroot_deployment_set_mutable (self, new_deployment, FALSE,
cancellable, error))
goto out;
if (!(self->debug_flags & OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS))
{
if (!ostree_sysroot_deployment_set_mutable (self, new_deployment, FALSE,
cancellable, error))
goto out;
}
{ ostree_cleanup_sepolicy_fscreatecon gpointer dummy = NULL;
src/libostree/ostree-sysroot-private.h
+8
View File
@@ -27,6 +27,13 @@
G_BEGIN_DECLS
typedef enum {
/* Don't flag deployments as immutable. */
OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS = 1 << 0
} OstreeSysrootDebugFlags;
struct OstreeSysroot {
GObject parent;
@@ -46,6 +53,7 @@ struct OstreeSysroot {
/* Only access through ostree_sysroot_get_repo() */
OstreeRepo *repo;
OstreeSysrootDebugFlags debug_flags;
};
#define OSTREE_SYSROOT_LOCKFILE "ostree/lock"
Loaded 100 of 166 files, more files were not shown because too many files have changed in this diff. Show more