diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 4458256e..00000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,121 +0,0 @@ -Submitting patches ------------------- - -You can: - - 1. Send mail to ostree-list@gnome.org, with the patch attached - 1. Submit a pull request against https://github.com/GNOME/ostree - 1. Attach them to https://bugzilla.gnome.org/ - -Please look at "git log" and match the commit log style. - -Running the test suite ----------------------- - -Currently, ostree uses https://wiki.gnome.org/GnomeGoals/InstalledTests -To run just ostree's tests: - - ./configure ... --enable-installed-tests - gnome-desktop-testing-runner -p 0 ostree/ - -Also, there is a regular: - - make check - -That runs a different set of tests. - -Coding style ------------- - -Indentation is GNU. Files should start with the appropriate mode lines. - -Use GCC `__attribute__((cleanup))` wherever possible. If interacting -with a third party library, try defining local cleanup macros. - -Use GError and GCancellable where appropriate. - -Prefer returning `gboolean` to signal success/failure, and have output -values as parameters. - -Prefer linear control flow inside functions (aside from standard -loops). In other words, avoid "early exits" or use of `goto` besides -`goto out;`. - -This is an example of an "early exit": - - static gboolean - myfunc (...) - { - gboolean ret = FALSE; - - /* some code */ - - /* some more code */ - - if (condition) - return FALSE; - - /* some more code */ - - ret = TRUE; - out: - return ret; - } - -If you must shortcut, use: - - if (condition) - { - ret = TRUE; - goto out; - } - -A consequence of this restriction is that you are encouraged to avoid -deep nesting of loops or conditionals. Create internal static helper -functions, particularly inside loops. For example, rather than: - - while (condition) - { - /* some code */ - if (condition) - { - for (i = 0; i < somevalue; i++) - { - if (condition) - { - /* deeply nested code */ - } - - /* more nested code */ - } - } - } - -Instead do this: - - static gboolean - helperfunc (..., GError **error) - { - if (condition) - { - /* deeply nested code */ - } - - /* more nested code */ - - return ret; - } - - while (condition) - { - /* some code */ - if (!condition) - continue; - - for (i = 0; i < somevalue; i++) - { - if (!helperfunc (..., i, error)) - goto out; - } - } - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 120000 index 00000000..49d1b98f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1 @@ +docs/CONTRIBUTING.md \ No newline at end of file diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 238cb0b4..23fa9494 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -182,7 +182,10 @@ pkgconfig_DATA += src/libostree/ostree-1.pc gpgreadme_DATA = src/libostree/README-gpg gpgreadmedir = $(pkgdatadir)/trusted.gpg.d -EXTRA_DIST += src/libostree/README-gpg src/libostree/bupsplit.h +EXTRA_DIST += src/libostree/README-gpg src/libostree/bupsplit.h \ + src/libostree/ostree-enumtypes.h.template \ + src/libostree/ostree-enumtypes.c.template \ + src/libostree/ostree-deployment-private.h install-mkdir-remotes-d-hook: mkdir -p $(DESTDIR)$(sysconfdir)/ostree/remotes.d diff --git a/Makefile-man.am b/Makefile-man.am new file mode 100644 index 00000000..615bf0f0 --- /dev/null +++ b/Makefile-man.am @@ -0,0 +1,58 @@ +# Makefile for man/ +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +if ENABLE_MAN + +man1_files = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-export.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 + +if BUILDOPT_FUSE +man1_files += rofiles-fuse.1 +endif + +man5_files = ostree.repo.5 ostree.repo-config.5 + +man1_MANS = $(addprefix man/,$(man1_files)) +man5_MANS = $(addprefix man/,$(man5_files)) + +EXTRA_DIST += $(man1_MANS) $(man5_MANS) + +XSLT_STYLESHEET = http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl + +XSLTPROC_FLAGS = \ + --nonet \ + --stringparam man.output.quietly 1 \ + --stringparam funcsynopsis.style ansi \ + --stringparam man.th.extra1.suppress 1 \ + --stringparam man.authors.section.enabled 0 \ + --stringparam man.copyright.section.enabled 0 + +XSLTPROC_MAN = $(XSLTPROC) $(XSLTPROC_FLAGS) + +%.1: %.xml + $(AM_V_GEN) $(XSLTPROC_MAN) --output $@ $(XSLT_STYLESHEET) $< + +%.5: %.xml + $(AM_V_GEN) $(XSLTPROC_MAN) --output $@ $(XSLT_STYLESHEET) $< + +CLEANFILES += \ + $(man1_MANS) \ + $(man5_MANS) \ + $(NULL) + +endif diff --git a/Makefile-ostree.am b/Makefile-ostree.am index 384479da..ff7e372b 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -28,6 +28,7 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-checksum.c \ src/ostree/ot-builtin-commit.c \ src/ostree/ot-builtin-diff.c \ + src/ostree/ot-builtin-export.c \ src/ostree/ot-builtin-fsck.c \ src/ostree/ot-builtin-gpg-sign.c \ src/ostree/ot-builtin-init.c \ @@ -89,6 +90,8 @@ ostree_SOURCES += \ src/ostree/parse-datetime.c: src/ostree/parse-datetime.y Makefile $(AM_V_GEN) $(YACC) $< -o $@ +EXTRA_DIST += src/ostree/parse-datetime.y +CLEANFILES += src/ostree/parse-datetime.c ostree_bin_shared_cflags = $(AM_CFLAGS) -I$(srcdir)/src/libotutil -I$(srcdir)/src/libostree -I$(srcdir)/src/ostree \ $(NULL) @@ -105,3 +108,8 @@ ostree_SOURCES += \ ostree_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) ostree_LDADD += $(OT_INTERNAL_SOUP_LIBS) endif + +if USE_LIBARCHIVE +ostree_CFLAGS += $(OT_DEP_LIBARCHIVE_CFLAGS) +ostree_LDADD += $(OT_DEP_LIBARCHIVE_LIBS) +endif diff --git a/Makefile-tests.am b/Makefile-tests.am index 9f3feef6..6ec58357 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -27,6 +27,7 @@ testfiles = test-basic \ test-remote-add \ test-remote-gpg-import \ test-commit-sign \ + test-export \ test-help \ test-libarchive \ test-pull-archive-z \ @@ -61,6 +62,11 @@ testfiles = test-basic \ test-auto-summary \ test-prune \ $(NULL) + +if BUILDOPT_FUSE +testfiles += test-rofiles-fuse +endif + insttest_SCRIPTS = $(addprefix tests/,$(testfiles:=.sh)) # This one uses corrupt-repo-ref.js @@ -79,6 +85,8 @@ insttest_DATA = tests/archive-test.sh \ tests/test-basic-user.sh \ tests/test-local-pull.sh \ tests/corrupt-repo-ref.js \ + tests/pre-endian-deltas-repo-big.tar.xz \ + tests/pre-endian-deltas-repo-little.tar.xz \ $(NULL) insttest_SCRIPTS += \ @@ -134,6 +142,10 @@ TESTS = tests/test-varint tests/test-ot-unix-utils tests/test-bsdiff tests/test- tests/test-keyfile-utils tests/test-ot-opt-utils tests/test-ot-tool-util \ tests/test-gpg-verify-result tests/test-checksum tests/test-lzma tests/test-rollsum +if USE_LIBARCHIVE +TESTS += tests/test-libarchive-import +endif + check_PROGRAMS = $(TESTS) TESTS_ENVIRONMENT = \ G_TEST_SRCDIR=$(abs_srcdir)/tests \ @@ -166,6 +178,10 @@ tests_test_checksum_SOURCES = src/libostree/ostree-core.c tests/test-checksum.c tests_test_checksum_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) tests_test_checksum_LDADD = $(TESTS_LDADD) +tests_test_libarchive_import_SOURCES = tests/test-libarchive-import.c +tests_test_libarchive_import_CFLAGS = $(TESTS_CFLAGS) $(libglnx_cflags) +tests_test_libarchive_import_LDADD = $(TESTS_LDADD) + tests_test_keyfile_utils_CFLAGS = $(TESTS_CFLAGS) tests_test_keyfile_utils_LDADD = $(TESTS_LDADD) diff --git a/Makefile.am b/Makefile.am index cc44166a..1625419b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,7 +32,7 @@ DISTCHECK_CONFIGURE_FLAGS += --enable-gtk-doc --disable-maintainer-mode SUBDIRS += . if ENABLE_GTK_DOC -SUBDIRS += doc +SUBDIRS += apidoc endif EXTRA_DIST += autogen.sh COPYING README.md @@ -43,8 +43,8 @@ OT_INTERNAL_SOUP_CFLAGS = $(OT_DEP_SOUP_CFLAGS) OT_INTERNAL_SOUP_LIBS = $(OT_DEP_SOUP_LIBS) # This canonicalizes the PKG_CHECK_MODULES or AM_PATH_GPGME results -OT_INTERNAL_GPGME_CFLAGS = $(OT_DEP_GPGME_CFLAGS) $(GPGME_CFLAGS) -OT_INTERNAL_GPGME_LIBS = $(OT_DEP_GPGME_LIBS) $(GPGME_LIBS) +OT_INTERNAL_GPGME_CFLAGS = $(OT_DEP_GPGME_CFLAGS) $(GPGME_PTHREAD_CFLAGS) +OT_INTERNAL_GPGME_LIBS = $(OT_DEP_GPGME_LIBS) $(GPGME_PTHREAD_LIBS) if BUILDOPT_INTROSPECTION include $(INTROSPECTION_MAKEFILE) @@ -68,8 +68,10 @@ include Makefile-otutil.am include Makefile-libostree.am include Makefile-ostree.am include Makefile-switchroot.am +include src/rofiles-fuse/Makefile-inc.am include Makefile-tests.am include Makefile-boot.am +include Makefile-man.am release-tag: git tag -m "Release $(VERSION)" v$(VERSION) diff --git a/README.md b/README.md index ac070da8..10f7c32f 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,48 @@ OSTree ====== +New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ ) + +----- + OSTree is a tool that combines a "git-like" model for committing and downloading bootable filesystem trees, along with a layer for deploying them and managing the bootloader configuration. -Traditional package managers (dpkg/rpm) build filesystem trees on the -client side. In contrast, the primary focus of OSTree is on -replicating trees composed on a server. +OSTree is like git in that it checksums individual files and has a +content-addressed-object store. It's unlike git in that it "checks +out" the files via hardlinks, and they should thus be immutable. +Therefore, another way to think of OSTree is that it's just a more +polished version of +[Linux VServer hardlinks](http://linux-vserver.org/index.php?title=util-vserver:Vhashify&oldid=2285). **Features:** - - Atomic upgrades and rollback - - GPG signatures and "pinned TLS" support + - Atomic upgrades and rollback for the system + - Replicating content incrementally over HTTP via GPG signatures and "pinned TLS" support - Support for parallel installing more than just 2 bootable roots - - Binary history on the server side + - Binary history on the server side (and client) - Introspectable shared library API for build and deployment systems +This last point is important - you should think of the OSTree command +line as effectively a "demo" for the shared library. The intent is that +package managers, system upgrade tools, container build tools and the like +use OSTree as a "deduplicating hardlink store". + Projects using OSTree --------------------- [rpm-ostree](https://github.com/projectatomic/rpm-ostree) is a tool that uses OSTree as a shared library, and supports committing RPMs -into an OSTree repository, and deploying them on the client. +into an OSTree repository, and deploying them on the client. This is +appropriate for "fixed purpose" systems. There is in progress work +for more sophisticated hybrid models, deeply integrating the RPM +packaging with OSTree. -[Project Atomic](http://www.projectatomic.io/) uses rpm-ostree -to provide a minimal host for Docker formatted Linux containers. +[Project Atomic](http://www.projectatomic.io/) uses rpm-ostree to +provide a minimal host for Docker formatted Linux containers. +Replicating a base immutable OS, then using Docker for applications +meshes together two different tools with different tradeoffs. [xdg-app](https://github.com/alexlarsson/xdg-app) uses OSTree for desktop application containers. @@ -46,8 +63,8 @@ versions support extended validation using However, in order to build from a git clone, you must update the submodules. If you're packaging OSTree and want a tarball, I recommend using a "recursive git archive" script. There are several -available online; [this -code](https://git.gnome.org/browse/ostree/tree/packaging/Makefile.dist-packaging#n11) +available online; +[this code](https://git.gnome.org/browse/ostree/tree/packaging/Makefile.dist-packaging#n11) in OSTree is an example. Once you have a git clone or recursive archive, building is the @@ -63,12 +80,11 @@ make install DESTDIR=/path/to/dest More documentation ------------------ +New! See the docs online at [Read The Docs (OSTree)](https://ostree.readthedocs.org/en/latest/ ) + Some more information is available on the old wiki page: https://wiki.gnome.org/Projects/OSTree -The intent is for that wiki page content to be migrated into Markdown -in this git repository. - Contributing ------------ diff --git a/doc/.gitignore b/apidoc/.gitignore similarity index 100% rename from doc/.gitignore rename to apidoc/.gitignore diff --git a/doc/Makefile.am b/apidoc/Makefile.am similarity index 71% rename from doc/Makefile.am rename to apidoc/Makefile.am index 9406f3ef..dcb009ef 100644 --- a/doc/Makefile.am +++ b/apidoc/Makefile.am @@ -95,11 +95,6 @@ HTML_IMAGES= # Extra SGML files that are included by $(DOC_MAIN_SGML_FILE). # e.g. content_files=running.sgml building.sgml changes-2.0.sgml content_files= \ - overview.xml \ - repo.xml \ - deployment.xml \ - atomic-upgrades.xml \ - adapting-existing.xml \ $(NULL) # SGML files where gtk-doc abbrevations (#GtkWidget) are expanded @@ -116,53 +111,13 @@ expand_content_files= \ # e.g. GTKDOC_LIBS=$(top_builddir)/gtk/$(gtktargetlib) GTKDOC_LIBS= -# Hacks around gtk-doc brokenness for out of tree builds -ostree-sections.txt: $(srcdir)/ostree-sections.txt - cp $< $@ - version.xml: echo -n $(VERSION) > "$@" # This includes the standard gtk-doc make rules, copied by gtkdocize. include $(top_srcdir)/gtk-doc.make -man1_MANS = ostree.1 ostree-admin-cleanup.1 ostree-admin-config-diff.1 ostree-admin-deploy.1 ostree-admin-init-fs.1 ostree-admin-instutil.1 ostree-admin-os-init.1 ostree-admin-status.1 ostree-admin-set-origin.1 ostree-admin-switch.1 ostree-admin-undeploy.1 ostree-admin-upgrade.1 ostree-admin.1 ostree-cat.1 ostree-checkout.1 ostree-checksum.1 ostree-commit.1 ostree-gpg-sign.1 ostree-config.1 ostree-diff.1 ostree-fsck.1 ostree-init.1 ostree-log.1 ostree-ls.1 ostree-prune.1 ostree-pull-local.1 ostree-pull.1 ostree-refs.1 ostree-remote.1 ostree-reset.1 ostree-rev-parse.1 ostree-show.1 ostree-summary.1 ostree-static-delta.1 ostree-trivial-httpd.1 - -man5_MANS = ostree.repo.5 ostree.repo-config.5 - -if ENABLE_GTK_DOC - -XSLTPROC_FLAGS = \ - --nonet \ - --stringparam man.output.quietly 1 \ - --stringparam funcsynopsis.style ansi \ - --stringparam man.th.extra1.suppress 1 \ - --stringparam man.authors.section.enabled 0 \ - --stringparam man.copyright.section.enabled 0 - -XSLTPROC_MAN = \ - $(XSLTPROC) $(XSLTPROC_FLAGS) http://docbook.sourceforge.net/release/xsl/current/manpages/docbook.xsl - -.xml.1: - $(AM_V_GEN) $(XSLTPROC_MAN) $< -.xml.5: - $(AM_V_GEN) $(XSLTPROC_MAN) $< - -CLEANFILES += \ - $(man1_MANS) \ - $(man5_MANS) \ - $(NULL) - -endif # ENABLE_GTK_DOC - -MAN_IN_FILES = \ - $(man1_MANS:.1=.xml) \ - $(man5_MANS:.5=.xml) \ - $(NULL) - EXTRA_DIST += \ - $(MAN_IN_FILES) \ version.xml \ - ostree.xml \ ostree-sections.txt \ $(NULL) diff --git a/doc/ostree-docs.xml b/apidoc/ostree-docs.xml similarity index 86% rename from doc/ostree-docs.xml rename to apidoc/ostree-docs.xml index d2e9f7b8..c5ea28e6 100644 --- a/doc/ostree-docs.xml +++ b/apidoc/ostree-docs.xml @@ -7,16 +7,10 @@ ]> - OSTree Manual + OSTree API references for OSTree &version; - - - - - - API Reference diff --git a/doc/ostree-sections.txt b/apidoc/ostree-sections.txt similarity index 99% rename from doc/ostree-sections.txt rename to apidoc/ostree-sections.txt index d448d6de..4b22e25b 100644 --- a/doc/ostree-sections.txt +++ b/apidoc/ostree-sections.txt @@ -220,6 +220,7 @@ ostree_repo_new_for_sysroot_path ostree_repo_new_default ostree_repo_open ostree_repo_set_disable_fsync +ostree_repo_get_disable_fsync ostree_repo_is_system ostree_repo_is_writable ostree_repo_create diff --git a/configure.ac b/configure.ac index b5bc1e56..0687ecdb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.63]) -AC_INIT([ostree], [2016.1], [walters@verbum.org]) +AC_INIT([ostree], [2016.3], [walters@verbum.org]) AC_CONFIG_HEADER([config.h]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_AUX_DIR([build-aux]) @@ -106,17 +106,19 @@ AM_CONDITIONAL(BUILDOPT_INTROSPECTION, test "x$found_introspection" = xyes) LIBGPGME_DEPENDENCY="1.1.8" -PKG_CHECK_MODULES(OT_DEP_GPGME, gpgme >= $LIBGPGME_DEPENDENCY, have_gpgme=yes, [ - m4_ifdef([AM_PATH_GPGME], [ - AM_PATH_GPGME($LIBGPGME_DEPENDENCY, have_gpgme=yes, have_gpgme=no) +PKG_CHECK_MODULES(OT_DEP_GPGME, gpgme-pthread >= $LIBGPGME_DEPENDENCY, have_gpgme=yes, [ + m4_ifdef([AM_PATH_GPGME_PTHREAD], [ + AM_PATH_GPGME_PTHREAD($LIBGPGME_DEPENDENCY, have_gpgme=yes, have_gpgme=no) ],[ have_gpgme=no ]) ]) AS_IF([ test x$have_gpgme = xno ], [ - AC_MSG_ERROR([Need GPGME version $LIBGPGME_DEPENDENCY or later]) + AC_MSG_ERROR([Need GPGME_PTHREAD version $LIBGPGME_DEPENDENCY or later]) ]) OSTREE_FEATURES="$OSTREE_FEATURES +gpgme" LIBARCHIVE_DEPENDENCY="libarchive >= 2.8.0" +# What's in RHEL7.2. +FUSE_DEPENDENCY="fuse >= 2.9.2" # check for gtk-doc m4_ifdef([GTK_DOC_CHECK], [ @@ -126,7 +128,22 @@ enable_gtk_doc=no AM_CONDITIONAL([ENABLE_GTK_DOC], false) ]) -AC_PATH_PROG([XSLTPROC], [xsltproc]) +AC_ARG_ENABLE(man, + [AS_HELP_STRING([--enable-man], + [generate man pages [default=auto]])],, + enable_man=maybe) + +AS_IF([test "$enable_man" != no], [ + AC_PATH_PROG([XSLTPROC], [xsltproc]) + AS_IF([test -z "$XSLTPROC"], [ + AS_IF([test "$enable_man" = yes], [ + AC_MSG_ERROR([xsltproc is required for --enable-man]) + ]) + enable_man=no + ]) + enable_man=yes +]) +AM_CONDITIONAL(ENABLE_MAN, test "$enable_man" != no) AC_ARG_WITH(libarchive, AS_HELP_STRING([--without-libarchive], [Do not use libarchive]), @@ -179,6 +196,16 @@ AS_IF([ test x$with_selinux != xno ], [ if test x$with_selinux != xno; then OSTREE_FEATURES="$OSTREE_FEATURES +selinux"; fi AM_CONDITIONAL(USE_SELINUX, test $with_selinux != no) +# Enabled by default because I think people should use it. +AC_ARG_ENABLE(rofiles-fuse, + [AS_HELP_STRING([--enable-rofiles-fuse], + [generate rofiles-fuse helper [default=yes]])],, + enable_rofiles_fuse=yes) +AS_IF([ test $enable_rofiles_fuse != xno ], [ + PKG_CHECK_MODULES(BUILDOPT_FUSE, $FUSE_DEPENDENCY) +], [enable_rofiles_fuse=no]) +AM_CONDITIONAL(BUILDOPT_FUSE, test x$enable_rofiles_fuse = xyes) + AC_ARG_WITH(dracut, AS_HELP_STRING([--with-dracut], [Install dracut module (default: no)]),, @@ -220,18 +247,9 @@ AS_IF([test "x$found_introspection" = xyes], [ ], [have_gjs=no]) AM_CONDITIONAL(BUILDOPT_GJS, test x$have_gjs = xyes) -AC_ARG_ENABLE(static_deltas, - AS_HELP_STRING([--enable-static-deltas], - [Enable static delta code (default: yes)]),, - [enable_static_deltas=yes]) -AS_IF([test x$enable_static_deltas = xyes], [ - AC_DEFINE([BUILDOPT_STATIC_DELTAS], 1, [Define if static deltas are enabled]) -]) -AM_CONDITIONAL(BUILDOPT_STATIC_DELTAS, test x$enable_static_deltas = xyes) - AC_CONFIG_FILES([ Makefile -doc/Makefile +apidoc/Makefile src/libostree/ostree-1.pc ]) AC_OUTPUT @@ -242,12 +260,14 @@ echo " introspection: $found_introspection + rofiles-fuse: $enable_rofiles_fuse libsoup (retrieve remote HTTP repositories): $with_soup libsoup TLS client certs: $have_libsoup_client_certs SELinux: $with_selinux libarchive (parse tar files directly): $with_libarchive - static deltas: $enable_static_deltas - documentation: $enable_gtk_doc + static deltas: yes (always enabled now) + man pages (xsltproc): $enable_man + api docs (gtk-doc): $enable_gtk_doc gjs-based tests: $have_gjs dracut: $with_dracut mkinitcpio: $with_mkinitcpio" diff --git a/doc/adapting-existing.xml b/doc/adapting-existing.xml deleted file mode 100644 index 5d1e0011..00000000 --- a/doc/adapting-existing.xml +++ /dev/null @@ -1,267 +0,0 @@ - - -]> - - Adapting existing mainstream distributions - - System layout - - First, OSTree encourages systems to implement UsrMove. - This is simply to avoid the need for more bind mounts. By - default OSTree's dracut hook creates a read-only bind mount over - /usr; you can of course - generate individual bind-mounts for /bin, all the /lib variants, etc. So it is not - intended to be a hard requirement. - - - - Remember, because by default the system is booted into a - chroot equivalent, there has to be some way - to refer to the actual physical root filesystem. Therefore, - your operating system tree should contain an empty /sysroot directory; at boot time, - OSTree will make this a bind mount to the physical / root - directory. There is precedent for this name in the initramfs - context. You should furthermore make a toplevel symbolic link - /ostree which points to - /sysroot/ostree, so that - the OSTree tool at runtime can consistently find the system data - regardless of whether it's operating on a physical root or - inside a deployment. - - - - Because OSTree only preserves /var across upgrades (each - deployment's chroot directory will be garbage collected - eventually), you will need to choose how to handle other - toplevel writable directories specified by the Filesystem Hierarchy - Standard. Your operating system may of course choose - not to support some of these such as /usr/local, but following is the - recommended set: - - - - /home to /var/home - - - - - /opt to /var/opt - - - - - /srv to /var/srv - - - - - /root to /var/roothome - - - - - /usr/local to /var/local - - - - - /mnt to /var/mnt - - - - - /tmp to /sysroot/tmp - - - - - - - Furthermore, since /var - is empty by default, your operating system will need to - dynamically create the targets of these at - boot. A good way to do this is using - systemd-tmpfiles, if your OS uses systemd. - For example: - - - - - - - - Particularly note here the double indirection of /home. By default, each - deployment will share the global toplevel /home directory on the physical - root filesystem. It is then up to higher levels of management - tools to keep /etc/passwd or equivalent - synchronized between operating systems. - - - Each deployment can easily be reconfigured to have its own home - directory set simply by making /var/home a real directory. - - - - - Booting and initramfs technology - - OSTree comes with optional dracut+systemd integration code that - parses the ostree= kernel command line - argument in the initramfs, and then sets up the read-only bind - mount on /usr, a bind - mount on the deployment's /sysroot to the physical /, and then finally uses - mount(MS_MOVE) to make the deployment root appear to be the - root filesystem before telling systemd to switch root. - - - - If you are not using dracut or systemd, using OSTree should still - be possible, but you will have to write the integration code. Patches - to support other initramfs technologies and init systems, if sufficiently - clean, will likely be accepted upstream. - - - - A further specific note regarding sysvinit: - OSTree used to support recording device files such the - /dev/initctl FIFO, but no longer does. - It's recommended to just patch your initramfs to create this at - boot. - - - - - /usr/lib/passwd - - Unlike traditional package systems, OSTree trees contain - numeric uid and gids. Furthermore, it does - not have a %post type mechanism where - useradd could be invoked. In order to ship - an OS that contains both system users and users dynamically - created on client machines, you will need to choose a solution - for /etc/passwd. The core problem is that - if you add a user to the system for a daemon, the OSTree upgrade - process for /etc will - simply notice that because /etc/passwd - differs from the previous default, it will keep the modified - config file, and your new OS user will not be visible. - - - The solution chosen for the gnome-continuous - operating system is to create - /usr/lib/passwd, and to include a NSS - module nss-altfiles - which instructs glibc to read from it. Then, the build system - places all system users there, freeing up - /etc/passwd to be purely a database of - local users. See also a more recent effort from Systemd - stateless. - - - - - Adapting existing package managers - - The largest endeavor is likely to be redesigning your - distribution's package manager to be on top of OSTree, - particularly if you want to keep compatibility with the "old - way" of installing into the physical /. This section will use examples - from both dpkg and rpm as - the author has familiarity with both; but the abstract concepts - should apply to most traditional package managers. - - - - There are many levels of possible integration; initially, we - will describe the most naive implementation which is the - simplest but also the least efficient. We will assume here that - the admin is booted into an OSTree-enabled system, and wants to - add a set of packages. - - - - Many package managers store their state in /var; but since in the OSTree model - that directory is shared between independent versions, the - package database must first be found in the per-deployment - /usr directory. It - becomes read-only; remember, all upgrades involve constructing a - new filesystem tree, so your package manager will also need to - create a copy of its database. Most likely, if you want to - continue supporting non-OSTree deployments, simply have your - package manager fall back to the legacy /var location if the one in - /usr is not found. - - - - To install a set of new packages (without removing any existing - ones), enumerate the set of packages in the currently booted - deployment, and perform dependency resolution to compute the - complete set of new packages. Download and unpack these new - packages to a temporary directory. - - - - Now, because we are merely installing new packages and not - removing anything, we can make the major optimization of reusing - our existing filesystem tree, and merely - layering the composed filesystem tree of - these new packages on top. A command lke this: ostree - commit -b osname/releasename/description - --tree=ref=osname/releasenamename/description - --tree=dir=/var/tmp/newpackages.13A8D0/ will create a - new commit in the - osname/releasename/description - branch. The OSTree SHA256 checksum of all the files in - /var/tmp/newpackages.13A8D0/ will be computed, but we will not - re-checksum the present existing tree. In this layering model, - earlier directories will take precedence, but files in later - layers will silently override earlier layers. - - - - Then to actually deploy this tree for the next boot: - ostree admin deploy - osname/releasenamename/description - - - - - diff --git a/doc/atomic-upgrades.xml b/doc/atomic-upgrades.xml deleted file mode 100644 index fd949fbf..00000000 --- a/doc/atomic-upgrades.xml +++ /dev/null @@ -1,181 +0,0 @@ - - -]> - - Atomic Upgrades - - You can turn off the power anytime you want... - - OSTree is designed to implement fully atomic and safe upgrades; - more generally, atomic transitions between lists of bootable - deployments. If the system crashes or you pull the power, you - will have either the old system, or the new one. - - - - - Simple upgrades via HTTP - - First, the most basic model OSTree supports is one where it - replicates pre-generated filesystem trees from a server over - HTTP, tracking exactly one ref, which is stored in the .origin file for the deployment. - The command ostree admin upgrade implements - this. - - - - To begin a simple upgrade, OSTree fetches the contents of the - ref from the remote server. Suppose we're tracking a ref named - exampleos/buildmaster/x86_64-runtime. - OSTree fetches the URL - http://example.com/repo/refs/exampleos/buildmaster/x86_64-runtime, - which contains a SHA256 checksum. This determines the tree to - deploy, and /etc will be - merged from currently booted tree. - - - - If we do not have this commit, then, then we perform a pull - process. At present (without static deltas), this involves - quite simply just fetching each individual object that we do not - have, asynchronously. Put in other words, we only download - changed files (zlib-compressed). Each object has its checksum - validated and is stored in /ostree/repo/objects/. - - - - Once the pull is complete, we have all the objects locally - we need to perform a deployment. - - - - - Upgrades via external tools (e.g. package managers) - - - As mentioned in the introduction, OSTree is also designed to - allow a model where filesystem trees are computed on the client. - It is completely agnostic as to how those trees are generated; - they could be computed with traditional packages, packages with - post-deployment scripts on top, or built by developers directly - from revision control locally, etc. - - - - At a practical level, most package managers today - (dpkg and rpm) operate - "live" on the currently booted filesystem. The way they could - work with OSTree is instead to take the list of installed - packages in the currently booted tree, and compute a new - filesystem from that. A later chapter describes in more details - how this could work: . - - - - For the purposes of this section, let's assume that we have a - newly generated filesystem tree stored in the repo (which shares - storage with the existing booted tree). We can then move on to - checking it back out of the repo into a deployment. - - - - - Assembling a new deployment directory - - Given a commit to deploy, OSTree first allocates a directory for - it. This is of the form /boot/loader/entries/ostree-osname-checksum.serial.conf. - The serial is normally 0, but if a - given commit is deployed more than once, it will be incremented. - This is supported because the previous deployment may have - configuration in /etc - that we do not want to use or overwrite. - - - - Now that we have a deployment directory, a 3-way merge is - performed between the (by default) currently booted deployment's - /etc, its default - configuration, and the new deployment (based on its /usr/etc). - - - - - Atomically swapping boot configuration - - At this point, a new deployment directory has been created as a - hardlink farm; the running system is untouched, and the - bootloader configuration is untouched. We want to add this deployment - to the "deployment list". - - - - To support a more general case, OSTree supports atomic - transitioning between arbitrary sets of deployments, with the - restriction that the currently booted deployment must always be - in the new set. In the normal case, we have exactly one - deployment, which is the booted one, and we want to add the new - deployment to the list. A more complex command might allow - creating 100 deployments as part of one atomic transaction, so - that one can set up an automated system to bisect across them. - - - - The bootversion - - OSTree allows swapping between boot configurations by - implementing the "swapped directory pattern" in /boot. This means it is a - symbolic link to one of two directories /ostree/boot.[0|1]. - To swap the contents atomically, if the current version is - 0, we create /ostree/boot.1, populate it with - the new contents, then atomically swap the symbolic link. Finally, - the old contents can be garbage collected at any point. - - - - - The /ostree/boot directory - - However, we want to optimize for the case where we the set of - kernel/initramfs pairs is the same between both the old and - new deployment lists. This happens when doing an upgrade that - does not include the kernel; think of a simple translation - update. OSTree optimizes for this case because on some - systems /boot may be on - a separate medium such as flash storage not optimized for - significant amounts of write traffic. - - - - To implement this, OSTree also maintains the directory - /ostree/boot.bootversion, - which is a set of symbolic links to the deployment - directories. The bootversion here - must match the version of /boot. However, in order to - allow atomic transitions of this - directory, this is also a swapped directory, so just like - /boot, it has a version - of 0 or 1 appended. - - - - Each bootloader entry has a special ostree= - argument which refers to one of these symbolic links. This is - parsed at runtime in the initramfs. - - - - - - - diff --git a/doc/deployment.xml b/doc/deployment.xml deleted file mode 100644 index 489850ae..00000000 --- a/doc/deployment.xml +++ /dev/null @@ -1,158 +0,0 @@ - - -]> - - Deployments - - Overview - - Built on top of the OSTree versioning filesystem core is a layer - that knows how to deploy, parallel install, and manage Unix-like - operating systems (accessible via ostree - admin). The core content of these operating systems - are treated as read-only, but they transparently share storage. - - - - A deployment is physically located at a path of the form - /ostree/deploy/osname/deploy/checksum. - OSTree is designed to boot directly into exactly one deployment - at a time; each deployment is intended to be a target for - chroot() or equivalent. - - - - - - "osname": Group of deployments that share /var - - Each deployment is grouped in exactly one "osname". From - above, you can see that an osname is physically represented in - the /ostree/deploy/osname - directory. For example, OSTree can allow parallel installing - Debian in /ostree/deploy/debian and Red Hat - Enterprise Linux in /ostree/deploy/rhel (subject to - operating system support, present released versions of these - operating systems may not support this). - - - - Each osname has exactly one copy of the traditional Unix - /var, stored physically - in /ostree/deploy/osname/var. - OSTree provides support tools for systemd - to create a Linux bind mount that ensures the booted - deployment sees the shared copy of /var. - - - - OSTree does not touch the contents of /var. Operating system components - such as daemon services are required to create any directories - they require there at runtime (e.g. /var/cache/daemonname), - and to manage upgrading data formats inside those directories. - - - - - Contents of a deployment - - A deployment begins with a specific commit (represented as a - SHA256 hash) in the OSTree repository in /ostree/repo. This commit refers - to a filesystem tree that represents the underlying basis of a - deployment. For short, we will call this the "tree", to - distinguish it from the concept of a deployment. - - - - First, the tree must include a kernel stored as /boot/vmlinuz-checksum. - The checksum should be a SHA256 hash of the kernel contents; - it must be pre-computed before storing the kernel in the - repository. Optionally, the tree can contain an initramfs, - stored as /boot/initramfs-checksum. - If this exists, the checksum must include both the kernel and - initramfs contents. OSTree will use this to determine which - kernels are shared. The rationale for this is to avoid - computing checksums on the client by default. - - - - The deployment should not have a traditional UNIX /etc; instead, it should include - /usr/etc. This is the - "default configuration". When OSTree creates a deployment, it - performs a 3-way merge using the old - default configuration, the active system's /etc, and the new default - configuration. In the final filesystem tree for a deployment - then, /etc is a regular - writable directory. - - - - Besides the exceptions of /var and /etc then, the rest of the - contents of the tree are checked out as hard links into the - repository. It's strongly recommended that operating systems - ship all of their content in /usr, but this is not a hard - requirement. - - - - Finally, a deployment may have a .origin file, stored next to its - directory. This file tells ostree admin - upgrade how to upgrade it. At the moment, OSTree only - supports upgrading a single refspec. However, in the future - OSTree may support a syntax for composing layers of trees, for - example. - - - - - - The system /boot - - While OSTree parallel installs deployments cleanly inside the - /ostree directory, - ultimately it has to control the system's /boot directory. The way this - works is via the boot - loader specification, which is a standard for - bootloader-independent drop-in configuration files. - - - When a tree is deployed, it will have a configuration file - generated of the form /boot/loader/entries/ostree-osname-checksum.serial.conf. - This configuration file will include a special - ostree= kernel argument that allows the - initramfs to find (and chroot() into) the - specified deployment. - - - - At present, not all bootloaders implement the BootLoaderSpec, - so OSTree contains code for some of these to regenerate native - config files (such as - /boot/syslinux/syslinux.conf based on the - entries. - - - - diff --git a/doc/overview.xml b/doc/overview.xml deleted file mode 100644 index 94bf9c78..00000000 --- a/doc/overview.xml +++ /dev/null @@ -1,155 +0,0 @@ - - -]> - - OSTree Overview - - Introduction - - OSTree an upgrade system for Linux-based operating systems that - performs atomic upgrades of complete filesystem trees. It is - not a package system; rather, it is intended to complement them. - A primary model is composing packages on a server, and then - replicating them to clients. - - - - The underlying architecture might be summarized as "git for - operating system binaries". It operates in userspace, and will - work on top of any Linux filesystem. At its core is a git-like - content-addressed object store, and layered on top of that is - bootloader configuration, management of - /etc, and other functions to perform an - upgrade beyond just replicating files. - - - - You can use OSTree standalone in the pure replication model, - but another approach is to add a package manager on top, - thus creating a hybrid tree/package system. - - - - - - Comparison with "package managers" - - Because OSTree is designed for deploying core operating - systems, a comparison with traditional "package managers" such - as dpkg and rpm is illustrative. Packages are traditionally - composed of partial filesystem trees with metadata and scripts - attached, and these are dynamically assembled on the client - machine, after a process of dependency resolution. - - - - In contrast, OSTree only supports recording and deploying - complete (bootable) filesystem trees. It - has no built-in knowledge of how a given filesystem tree was - generated or the origin of individual files, or dependencies, - descriptions of individual components. Put another way, OSTree - only handles delivery and deployment; you will likely still want - to include inside each tree metadata about the individual - components that went into the tree. For example, a system - administrator may want to know what version of OpenSSL was - included in your tree, so you should support the equivalent of - rpm -q or dpkg -L. - - - - The OSTree core emphasizes replicating read-only OS trees via - HTTP, and where the OS includes (if desired) an entirely - separate mechanism to install applications, stored in /var if they're system global, or - /home for per-user - application installation. An example application mechanism is - Docker. - - - - However, it is entirely possible to use OSTree underneath a - package system, where the contents of /usr are computed on the client. - For example, when installing a package, rather than changing the - currently running filesystem, the package manager could assemble - a new filesystem tree that layers the new packages on top of a - base tree, record it in the local OSTree repository, and then - set it up for the next boot. To support this model, OSTree - provides an (introspectable) C shared library. - - - - - Comparison with block/image replication - - OSTree shares some similarity with "dumb" replication and - stateless deployments, such as the model common in "cloud" - deployments where nodes are booted from an (effectively) - readonly disk, and user data is kept on a different volumes. - The advantage of "dumb" replication, shared by both OSTree and - the cloud model, is that it's reliable - and predictable. - - - But unlike many default image-based deployments, OSTree supports - exactly two persistent writable directories that are preserved - across upgrades: /etc and - /var. - - - Because OSTree operates at the Unix filesystem layer, it works - on top of any filesystem or block storage layout; it's possible - to replicate a given filesystem tree from an OSTree repository - into plain ext4, BTRFS, XFS, or in general any Unix-compatible - filesystem that supports hard links. Note: OSTree will - transparently take advantage of some BTRFS features if deployed - on it. - - - - - Atomic transitions between parallel-installable read-only filesystem trees - - Another deeply fundamental difference between both package - managers and image-based replication is that OSTree is - designed to parallel-install multiple - versions of multiple - independent operating systems. OSTree - relies on a new toplevel ostree directory; it can in fact - parallel install inside an existing OS or distribution - occupying the physical / root. - - - On each client machine, there is an OSTree repository stored - in /ostree/repo, and a - set of "deployments" stored in /ostree/deploy/OSNAME/CHECKSUM. - Each deployment is primarily composed of a set of hardlinks - into the repository. This means each version is deduplicated; - an upgrade process only costs disk space proportional to the - new files, plus some constant overhead. - - - The model OSTree emphasizes is that the OS read-only content - is kept in the classic Unix /usr; it comes with code to - create a Linux read-only bind mount to prevent inadvertent - corruption. There is exactly one /var writable directory shared - between each deployment for a given OS. The OSTree core code - does not touch content in this directory; it is up to the code - in each operating system for how to manage and upgrade state. - - - Finally, each deployment has its own writable copy of the - configuration store /etc. On upgrade, OSTree will - perform a basic 3-way diff, and apply any local changes to the - new copy, while leaving the old untouched. - - - diff --git a/doc/repo.xml b/doc/repo.xml deleted file mode 100644 index 5379d3a0..00000000 --- a/doc/repo.xml +++ /dev/null @@ -1,127 +0,0 @@ - - -]> - - Anatomy of an OSTree repository - - Core object types and data model - - OSTree is deeply inspired by git; the core layer is a userspace - content-addressed versioning filesystem. It is worth taking - some time to familiarize yourself with Git - Internals, as this section will assume some knowledge of - how git works. - - - - Its object types are similar to git; it has commit objects and - content objects. Git has "tree" objects, whereas OSTree splits - them into "dirtree" and "dirmeta" objects. But unlike git, - OSTree's checksums are SHA256. And most crucially, its content - objects include uid, gid, and extended attributes (but still no - timestamps). - - - - Commit objects - - A commit object contains metadata such as a timestamp, a log - message, and most importantly, a reference to a - dirtree/dirmeta pair of checksums which describe the root - directory of the filesystem. - - - Also like git, each commit in OSTree can have a parent. It is - designed to store a history of your binary builds, just like git - stores a history of source control. However, OSTree also makes - it easy to delete data, under the assumption that you can - regenerate it from source code. - - - - - Dirtree objects - - A dirtree contains a sorted array of (filename, checksum) - pairs for content objects, and a second sorted array of - (filename, dirtree checksum, dirmeta checksum), which are - subdirectories. - - - - - Dirmeta objects - - In git, tree objects contain the metadata such as permissions - for their children. But OSTree splits this into a separate - object to avoid duplicating extended attribute listings. - - - - - Content objects - - Unlike the first three object types which are metadata, - designed to be mmap()ed, the content object - has a separate internal header and payload sections. The - header contains uid, gid, mode, and symbolic link target (for - symlinks), as well as extended attributes. After the header, - for regular files, the content follows. - - - - - - Repository types and locations - - - Also unlike git, an OSTree repository can be in one of two - separate modes: bare and - archive-z2. A bare repository is one where - content files are just stored as regular files; it's designed to - be the source of a "hardlink farm", where each operating system - checkout is merely links into it. If you want to store files - owned by e.g. root in this mode, you must run OSTree as root. - In contrast, the archive-z2 mode is designed - for serving via plain HTTP. Like tar files, it can be - read/written by non-root users. - - - - On an OSTree-deployed system, the "system repository" is - /ostree/repo. It can be - read by any uid, but only written by root. Unless the - --repo argument is given to the - ostree command, it will operate on the system - repository. - - - - - Refs - - Like git, OSTree uses "refs" to which are text files that point - to particular commits (i.e. filesystem trees). For example, the - gnome-ostree operating system creates trees named like - gnome-ostree/buildmaster/x86_64-runtime and - gnome-ostree/buildmaster/x86_64-devel-debug. - These two refs point to two different generated filesystem - trees. In this example, the "runtime" tree contains just enough - to run a basic GNOME system, and "devel-debug" contains all of - the developer tools. - - - - The ostree supports a simple syntax using the - carat ^ to refer to the parent of a given - commit. For example, - gnome-ostree/buildmaster/x86_64-runtime^ - refers to the previous build, and - gnome-ostree/buildmaster/x86_64-runtime^^ - refers to the one before that. - - - diff --git a/docs-md/atomic-upgrades.md b/docs-md/atomic-upgrades.md deleted file mode 100644 index 76cd0250..00000000 --- a/docs-md/atomic-upgrades.md +++ /dev/null @@ -1,12 +0,0 @@ -Atomic upgrades and rollback ----------------------------- - -Traditional package managers operate "live" on the running system. -This means it's possible for interrupted updates to result in a -half-updated system. This model also makes it significantly harder to -support rollbacks when updates fail. - -In contrast, OSTree always creates a *new* root whenever it's -performing an update. This new root shares storage via hardlinks with -the current system. Upon success, the bootloader configuration will -be updated. diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md new file mode 100644 index 00000000..4458256e --- /dev/null +++ b/docs/CONTRIBUTING.md @@ -0,0 +1,121 @@ +Submitting patches +------------------ + +You can: + + 1. Send mail to ostree-list@gnome.org, with the patch attached + 1. Submit a pull request against https://github.com/GNOME/ostree + 1. Attach them to https://bugzilla.gnome.org/ + +Please look at "git log" and match the commit log style. + +Running the test suite +---------------------- + +Currently, ostree uses https://wiki.gnome.org/GnomeGoals/InstalledTests +To run just ostree's tests: + + ./configure ... --enable-installed-tests + gnome-desktop-testing-runner -p 0 ostree/ + +Also, there is a regular: + + make check + +That runs a different set of tests. + +Coding style +------------ + +Indentation is GNU. Files should start with the appropriate mode lines. + +Use GCC `__attribute__((cleanup))` wherever possible. If interacting +with a third party library, try defining local cleanup macros. + +Use GError and GCancellable where appropriate. + +Prefer returning `gboolean` to signal success/failure, and have output +values as parameters. + +Prefer linear control flow inside functions (aside from standard +loops). In other words, avoid "early exits" or use of `goto` besides +`goto out;`. + +This is an example of an "early exit": + + static gboolean + myfunc (...) + { + gboolean ret = FALSE; + + /* some code */ + + /* some more code */ + + if (condition) + return FALSE; + + /* some more code */ + + ret = TRUE; + out: + return ret; + } + +If you must shortcut, use: + + if (condition) + { + ret = TRUE; + goto out; + } + +A consequence of this restriction is that you are encouraged to avoid +deep nesting of loops or conditionals. Create internal static helper +functions, particularly inside loops. For example, rather than: + + while (condition) + { + /* some code */ + if (condition) + { + for (i = 0; i < somevalue; i++) + { + if (condition) + { + /* deeply nested code */ + } + + /* more nested code */ + } + } + } + +Instead do this: + + static gboolean + helperfunc (..., GError **error) + { + if (condition) + { + /* deeply nested code */ + } + + /* more nested code */ + + return ret; + } + + while (condition) + { + /* some code */ + if (!condition) + continue; + + for (i = 0; i < somevalue; i++) + { + if (!helperfunc (..., i, error)) + goto out; + } + } + diff --git a/docs/index.md b/docs/index.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/docs/manual/adapting-existing.md b/docs/manual/adapting-existing.md new file mode 100644 index 00000000..858c82d3 --- /dev/null +++ b/docs/manual/adapting-existing.md @@ -0,0 +1,159 @@ +# Adapting existing mainstream distributions + +## System layout + +First, OSTree encourages systems to implement +[UsrMove](http://www.freedesktop.org/wiki/Software/systemd/TheCaseForTheUsrMerge/) +This is simply to avoid the need for more bind mounts. By default +OSTree's dracut hook creates a read-only bind mount over `/usr`; you +can of course generate individual bind-mounts for `/bin`, all the +`/lib` variants, etc. So it is not intended to be a hard requirement. + +Remember, because by default the system is booted into a `chroot` +equivalent, there has to be some way to refer to the actual physical +root filesystem. Therefore, your operating system tree should contain +an empty `/sysroot` directory; at boot time, OSTree will make this a +bind mount to the physical / root directory. There is precedent for +this name in the initramfs context. You should furthermore make a +toplevel symbolic link `/ostree` which points to `/sysroot/ostree`, so +that the OSTree tool at runtime can consistently find the system data +regardless of whether it's operating on a physical root or inside a +deployment. + +Because OSTree only preserves `/var` across upgrades (each +deployment's chroot directory will be garbage collected +eventually), you will need to choose how to handle other +toplevel writable directories specified by the [Filesystem Hierarchy Standard](http://www.pathname.com/fhs/") +Your operating system may of course choose +not to support some of these such as `/usr/local`, but following is the +recommended set: + + - `/home` → `/var/home` + - `/opt` → `/var/opt` + - `/srv` → `/var/srv` + - `/root` → `/var/roothome` + - `/usr/local` → `/var/local` + - `/mnt` → `/var/mnt` + - `/tmp` → `/sysroot/tmp` + +Furthermore, since `/var` is empty by default, your operating system +will need to dynamically create the targets of +these at boot. A good way to do this is using `systemd-tmpfiles`, if +your OS uses systemd. For example: + +``` +d /var/log/journal 0755 root root - +L /var/home - - - - ../sysroot/home +d /var/opt 0755 root root - +d /var/srv 0755 root root - +d /var/roothome 0700 root root - +d /var/usrlocal 0755 root root - +d /var/usrlocal/bin 0755 root root - +d /var/usrlocal/etc 0755 root root - +d /var/usrlocal/games 0755 root root - +d /var/usrlocal/include 0755 root root - +d /var/usrlocal/lib 0755 root root - +d /var/usrlocal/man 0755 root root - +d /var/usrlocal/sbin 0755 root root - +d /var/usrlocal/share 0755 root root - +d /var/usrlocal/src 0755 root root - +d /var/mnt 0755 root root - +d /run/media 0755 root root - +``` + +Particularly note here the double indirection of `/home`. By default, +each deployment will share the global toplevel `/home` directory on +the physical root filesystem. It is then up to higher levels of +management tools to keep /etc/passwd or +equivalent synchronized between operating systems. Each deployment +can easily be reconfigured to have its own home directory set simply +by making `/var/home` a real directory. + +## Booting and initramfs technology + +OSTree comes with optional dracut+systemd integration code that parses +the `ostree=` kernel command line argument in the initramfs, and then +sets up the read-only bind mount on `/usr`, a bind mount on the +deployment's `/sysroot` to the physical `/`, and then finally uses +`mount(MS_MOVE)` to make the deployment root appear to be the root +filesystem before telling systemd to switch root. + +If you are not using dracut or systemd, using OSTree should still be +possible, but you will have to write the integration code. Patches to +support other initramfs technologies and init systems, if sufficiently +clean, will likely be accepted upstream. + +A further specific note regarding `sysvinit`: OSTree used to support +recording device files such the `/dev/initctl` FIFO, but no longer +does. It's recommended to just patch your initramfs to create this at +boot. + +## /usr/lib/passwd + +Unlike traditional package systems, OSTree trees contain *numeric* uid +and gids. Furthermore, it does not have a `%post` type mechanism +where `useradd` could be invoked. In order to ship an OS that +contains both system users and users dynamically created on client +machines, you will need to choose a solution for `/etc/passwd`. The +core problem is that if you add a user to the system for a daemon, the +OSTree upgrade process for `/etc` will simply notice that because +`/etc/passwd` differs from the previous default, it will keep the +modified config file, and your new OS user will not be visible. The +solution chosen for the [Gnome Continuous](https://live.gnome.org/Projects/GnomeContinuous) operating +system is to create `/usr/lib/passwd`, and to include a NSS module +[nss-altfiles](https://github.com/aperezdc/nss-altfiles) which +instructs glibc to read from it. Then, the build system places all +system users there, freeing up `/etc/passwd` to be purely a database +of local users. See also a more recent effort from [Systemd Stateless](http://0pointer.de/blog/projects/stateless.html) + +## Adapting existing package managers + +The largest endeavor is likely to be redesigning your distribution's +package manager to be on top of OSTree, particularly if you want to +keep compatibility with the "old way" of installing into the physical +`/`. This section will use examples from both `dpkg` and `rpm` as the +author has familiarity with both; but the abstract concepts should +apply to most traditional package managers. + +There are many levels of possible integration; initially, we will +describe the most naive implementation which is the simplest but also +the least efficient. We will assume here that the admin is booted +into an OSTree-enabled system, and wants to add a set of packages. + +Many package managers store their state in `/var`; but since in the +OSTree model that directory is shared between independent versions, +the package database must first be found in the per-deployment `/usr` +directory. It becomes read-only; remember, all upgrades involve +constructing a new filesystem tree, so your package manager will also +need to create a copy of its database. Most likely, if you want to +continue supporting non-OSTree deployments, simply have your package +manager fall back to the legacy `/var` location if the one in `/usr` +is not found. + +To install a set of new packages (without removing any existing ones), +enumerate the set of packages in the currently booted deployment, and +perform dependency resolution to compute the complete set of new +packages. Download and unpack these new packages to a temporary +directory. + +Now, because we are merely installing new packages and not +removing anything, we can make the major optimization of reusing +our existing filesystem tree, and merely +*layering* the composed filesystem tree of +these new packages on top. A command like this: + +``` +ostree commit -b osname/releasename/description +--tree=ref=$osname/releasename/description +--tree=dir=/var/tmp/newpackages.13A8D0/ +``` + +will create a new commit in the `$osname/releasename/description` +branch. The OSTree SHA256 checksum of all the files in +`/var/tmp/newpackages.13A8D0/` will be computed, but we will not +re-checksum the present existing tree. In this layering model, +earlier directories will take precedence, but files in later layers +will silently override earlier layers. + +Then to actually deploy this tree for the next boot: +`ostree admin deploy osname/releasename/description` diff --git a/docs/manual/atomic-upgrades.md b/docs/manual/atomic-upgrades.md new file mode 100644 index 00000000..9ce2c8ae --- /dev/null +++ b/docs/manual/atomic-upgrades.md @@ -0,0 +1,116 @@ +# Atomic Upgrades + +## You can turn off the power anytime you want... + +OSTree is designed to implement fully atomic and safe upgrades; +more generally, atomic transitions between lists of bootable +deployments. If the system crashes or you pull the power, you +will have either the old system, or the new one. + +## Simple upgrades via HTTP + +First, the most basic model OSTree supports is one where it replicates +pre-generated filesystem trees from a server over HTTP, tracking +exactly one ref, which is stored in the `.origin` file for the +deployment. The command `ostree admin upgrade` +implements this. + +o begin a simple upgrade, OSTree fetches the contents of the ref from +the remote server. Suppose we're tracking a ref named +`exampleos/buildmaster/x86_64-runtime`. OSTree fetches the URL +`http://$example.com/repo/refs/exampleos/buildmaster/x86_64-runtime`, +which contains a SHA256 checksum. This determines the tree to deploy, +and `/etc` will be merged from currently booted tree. + +If we do not have this commit, then, then we perform a pull process. +At present (without static deltas), this involves quite simply just +fetching each individual object that we do not have, asynchronously. +Put in other words, we only download changed files (zlib-compressed). +Each object has its checksum validated and is stored in `/ostree/repo/objects/`. + +Once the pull is complete, we have all the objects locally +we need to perform a deployment. + +## Upgrades via external tools (e.g. package managers) + +As mentioned in the introduction, OSTree is also designed to allow a +model where filesystem trees are computed on the client. It is +completely agnostic as to how those trees are generated; hey could be +computed with traditional packages, packages with post-deployment +scripts on top, or built by developers directly from revision control +locally, etc. + +At a practical level, most package managers today (`dpkg` and `rpm`) +operate "live" on the currently booted filesystem. The way they could +work with OSTree is instead to take the list of installed packages in +the currently booted tree, and compute a new filesystem from that. A +later chapter describes in more details how this could work: +[adapting-existing.md](Adapting Existing Systems). + +For the purposes of this section, let's assume that we have a +newly generated filesystem tree stored in the repo (which shares +storage with the existing booted tree). We can then move on to +checking it back out of the repo into a deployment. + +## Assembling a new deployment directory + +Given a commit to deploy, OSTree first allocates a directory for +it. This is of the form `/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. +he $serial is normally 0, but if a +given commit is deployed more than once, it will be incremented. +his is supported because the previous deployment may have +configuration in `/etc` +hat we do not want to use or overwrite. + +Now that we have a deployment directory, a 3-way merge is +performed between the (by default) currently booted deployment's +`/etc`, its default +configuration, and the new deployment (based on its `/usr/etc`). + +## Atomically swapping boot configuration + +At this point, a new deployment directory has been created as a +hardlink farm; the running system is untouched, and the bootloader +configuration is untouched. We want to add this deployment o the +"deployment list". + +To support a more general case, OSTree supports atomic ransitioning +between arbitrary sets of deployments, with the restriction that the +currently booted deployment must always be in the new set. In the +normal case, we have exactly one deployment, which is the booted one, +and we want to add the new deployment to the list. A more complex +command might allow creating 100 deployments as part of one atomic +transaction, so that one can set up an automated system to bisect +across them. + +## The bootversion + +OSTree allows swapping between boot configurations by implementing the +"swapped directory pattern" in `/boot`. This means it is a symbolic +link to one of two directories `/ostree/boot.[0|1]`. To swap the +contents atomically, if the current version is `0`, we create +`/ostree/boot.1`, populate it with the new contents, then atomically +swap the symbolic link. Finally, the old contents can be garbage +collected at any point. + +## The /ostree/boot directory + +However, we want to optimize for the case where we the set of +kernel/initramfs pairs is the same between both the old and new +deployment lists. This happens when doing an upgrade that does not +include the kernel; think of a simple translation update. OSTree +optimizes for this case because on some systems `/boot` may be on a +separate medium such as flash storage not optimized for significant +amounts of write traffic. + +To implement this, OSTree also maintains the directory +`/ostree/boot.bootversion`, which is a set +of symbolic links to the deployment directories. The +bootversion here must match the version of +`/boot`. However, in order to allow atomic transitions of +this directory, this is also a swapped directory, +so just like `/boot`, it has a version of `0` or `1` appended. + +Each bootloader entry has a special `ostree=` argument which refers to +one of these symbolic links. This is parsed at runtime in the +initramfs. diff --git a/docs/manual/deployment.md b/docs/manual/deployment.md new file mode 100644 index 00000000..53b0b662 --- /dev/null +++ b/docs/manual/deployment.md @@ -0,0 +1,90 @@ +# Deployments + +## Overview + +Built on top of the OSTree versioning filesystem core is a layer +that knows how to deploy, parallel install, and manage Unix-like +operating systems (accessible via `ostree admin`). The core content of these operating systems +are treated as read-only, but they transparently share storage. + +A deployment is physically located at a path of the form +`/ostree/deploy/$osname/deploy/$checksum`. +OSTree is designed to boot directly into exactly one deployment +at a time; each deployment is intended to be a target for +`chroot()` or equivalent. + +### "osname": Group of deployments that share /var + +Each deployment is grouped in exactly one "osname". From above, you +can see that an osname is physically represented in the +`/ostree/deploy/$osname` directory. For example, OSTree can allow +parallel installing Debian in `/ostree/deploy/debian` and Red Hat +Enterprise Linux in `/ostree/deploy/rhel` (subject to operating system +support, present released versions of these operating systems may not +support this). + +Each osname has exactly one copy of the traditional Unix `/var`, +stored physically in `/ostree/deploy/$osname/var`. OSTree provides +support tools for `systemd` to create a Linux bind mount that ensures +the booted deployment sees the shared copy of `/var`. + +OSTree does not touch the contents of `/var`. Operating system +components such as daemon services are required to create any +directories they require there at runtime +(e.g. `/var/cache/$daemonname`), and to manage upgrading data formats +inside those directories. + +### Contents of a deployment + +A deployment begins with a specific commit (represented as a +SHA256 hash) in the OSTree repository in `/ostree/repo`. This commit refers +to a filesystem tree that represents the underlying basis of a +deployment. For short, we will call this the "tree", to +distinguish it from the concept of a deployment. + +First, the tree must include a kernel stored as +`/boot/vmlinuz-$checksum`. The checksum should be a SHA256 hash of +the kernel contents; it must be pre-computed before storing the kernel +in the repository. Optionally, the tree can contain an initramfs, +stored as `/boot/initramfs-$checksum`. If this exists, the checksum +must include both the kernel and initramfs contents. OSTree will use +this to determine which kernels are shared. The rationale for this is +to avoid computing checksums on the client by default. + +The deployment should not have a traditional UNIX `/etc`; instead, it +should include `/usr/etc`. This is the "default configuration". When +OSTree creates a deployment, it performs a 3-way merge using the +old default configuration, the active system's +`/etc`, and the new default configuration. In the final filesystem +tree for a deployment then, `/etc` is a regular writable directory. + +Besides the exceptions of `/var` and `/etc` then, the rest of the +contents of the tree are checked out as hard links into the +repository. It's strongly recommended that operating systems ship all +of their content in `/usr`, but this is not a hard requirement. + +Finally, a deployment may have a `.origin` file, stored next to its +directory. This file tells `ostree admin upgrade` how to upgrade it. +At the moment, OSTree only supports upgrading a single refspec. +However, in the future OSTree may support a syntax for composing +layers of trees, for example. + +### The system /boot + +While OSTree parallel installs deployments cleanly inside the +`/ostree` directory, ultimately it has to control the system's `/boot` +directory. The way this works is via the +[Boot Loader Specification](http://www.freedesktop.org/wiki/Specifications/BootLoaderSpec), +which is a standard for bootloader-independent drop-in configuration +files. + +When a tree is deployed, it will have a configuration file generated +of the form +`/boot/loader/entries/ostree-$osname-$checksum.$serial.conf`. This +configuration file will include a special `ostree=` kernel argument +that allows the initramfs to find (and `chroot()` into) the specified +deployment. + +At present, not all bootloaders implement the BootLoaderSpec, so +OSTree contains code for some of these to regenerate native config +files (such as `/boot/syslinux/syslinux.conf` based on the entries. diff --git a/docs/manual/formats.md b/docs/manual/formats.md new file mode 100644 index 00000000..bf7fd0ae --- /dev/null +++ b/docs/manual/formats.md @@ -0,0 +1,181 @@ +# OSTree data formats + +## On the topic of "smart servers" + +One really crucial difference between OSTree and git is that git has a +"smart server". Even when fetching over `https://`, it isn't just a +static webserver, but one that e.g. dynamically computes and +compresses pack files for each client. + +In contrast, the author of OSTree feels that for operating system +updates, many deployments will want to use simple static webservers, +the same target most package systems were designed to use. The +primary advantages are security and compute efficiency. Services like +Amazon S3 and CDNs are a canonical target, as well as a stock static +nginx server. + +## The archive-z2 format + +In the [repo](repo) section, the concept of objects was introduced, +where file/content objects are checksummed and managed individually. +(Unlike a package system, which operates on compressed aggregates). + +The archive-z2 format simply gzip-compresses each content object. +Metadata objects are stored uncompressed. This means that it's easy +to serve via static HTTP. + +When you commit new content, you will see new `.filez` files appearing +in `objects/`. + +## archive-z2 efficiency + +The advantages of `archive-z2`: + + - It's easy to understand and implement + - Can be served directly over plain HTTP by a static webserver + - Clients can download/unpack updates incrementally + - Space efficient on the server + +The biggest disadvantage of this format is that for a client to +perform an update, one HTTP request per changed file is required. In +some scenarios, this actually isn't bad at all, particularly with +techniques to reduce HTTP overhead, such as +[HTTP/2](https://en.wikipedia.org/wiki/HTTP/2). + +In order to make this format work well, you should design your content +such that large data that changes infrequently (e.g. graphic images) +are stored separately from small frequently changing data (application +code). + +Other disadvantages of `archive-z2`: + + - It's quite bad when clients are performing an initial pull (without HTTP/2), + - One doesn't know the total size (compressed or uncompressed) of content + before downloading everything + +## Aside: the bare and bare-user formats + +The most common operation is to pull from an `archive-z2` repository +into a `bare` or `bare-user` formatted repository. These latter two +are not compressed on disk. In other words, pulling to them is +similar to unpacking (but not installing) an RPM/deb package. + +The `bare-user` format is a bit special in that the uid/gid and xattrs +from the content are ignored. This is primarily useful if you want to +have the same OSTree-managed content that can be run on a host system +or an unprivileged container. + +## Static deltas + +OSTree itself was originally focused on a continous delivery model, where +client systems are expected to update regularly. However, many OS vendors +would like to supply content that's updated e.g. once a month or less often. + +For this model, we can do a lot better to support batched updates than +a basic `archive-z2` repo. However, we still want to preserve the +model of "static webserver only". Given this, OSTree has gained the +concept of a "static delta". + +These deltas are targeted to be a delta between two specific commit +objects, including "bsdiff" and "rsync-style" deltas within a content +object. Static deltas also support `from NULL`, where the client can +more efficiently download a commit object from scratch. + +Effectively, we're spending server-side storage (and one-time compute +cost), and gaining efficiency in client network bandwith. + +## Static delta repository layout + +Since static deltas may not exist, the client first needs to attempt +to locate one. Suppose a client wants to retrieve commit `${new}` +while currently running `${current}`. + +The first thing to understand is that in order to save space, these +two commits are "modified base64" - the `/` character is replaced with +`_`. + +Like the commit objects, a "prefix directory" is used to make +management easier for filesystem tools + +A delta is named `$(mbase64 $from)-$(mbase64 $to)`, for example +`GpTyZaVut2jXFPWnO4LJiKEdRTvOw_mFUCtIKW1NIX0-L8f+VVDkEBKNc1Ncd+mDUrSVR4EyybQGCkuKtkDnTwk`, +which in sha256 format is +`1a94f265a56eb768d714f5a73b82c988a11d453bcec3f985502b48296d4d217d-2fc7fe5550e410128d73535c77e98352b495478132c9b4060a4b8ab640e74f09`. + +Finally, the actual content can be found in +`deltas/$fromprefix/$fromsuffix-$to`. + +## Static delta internal structure + +A delta is itself a directory. Inside, there is a file called +`superblock` which contains metadata. The rest of the files will be +integers bearing packs of content. + +The file format of static deltas should be currently considered an +OSTree implementation detail. Obviously, nothing stops one from +writing code which is compatible with OSTree today. However, we would +like the flexibility to expand and change things, and having multiple +codebases makes that more problematic. Please contact the authors +with any requests. + +That said, one critical thing to understand about the design is that +delta payloads are a bit more like "restricted programs" than they are +raw data. There's a "compilation" phase which generates output that +the client executes. + +This "updates as code" model allows for multiple content generation +strategies. The design of this was inspired by that of Chromium: +[http://dev.chromium.org/chromium-os/chromiumos-design-docs/filesystem-autoupdate](ChromiumOS +autoupdate). + +### The delta superblock + +The superblock contains: + + - arbitrary metadata + - delta generation timestamp + - the new commit object + - An array of recursive deltas to apply + - An array of per-part metadata, including total object sizes (compressed and uncompressed), + - An array of fallback objects + +Let's define a delta part, then return to discuss details: + +## A delta part + +A delta part is a combination of a raw blob of data, plus a very +restricted bytecode that operates on it. Say for example two files +happen to share a common section. It's possible for the delta +compilation to include that section once in the delta data blob, then +generate instructions to write out that blob twice when generating +both objects. + +Realistically though, it's very common for most of a delta to just be +"stream of new objects" - if one considers it, it doesn't make sense +to have too much duplication inside operating system content at this +level. + +So then, what's more interesting is that OSTree static deltas support +a per-file delta algorithm called +[bsdiff](https://github.com/mendsley/bsdiff) that most notably works +well on executable code. + +The current delta compiler scans for files with maching basenamesin +each commit that have a similar size, and attempts a bsdiff between +them. (It would make sense later to have a build system provide a +hint for this - for example, files within a same package). + +A generated bsdiff is included in the payload blob, and applying it is +an instruction. + +## Fallback objects + +It's possible for there to be large-ish files which might be resistant +to bsdiff. A good example is that it's common for operating systems +to use an "initramfs", which is itself a compressed filesystem. This +"internal compression" defeats bsdiff analysis. + +For these types of objects, the delta superblock contains an array of +"fallback objects". These objects aren't included in the delta +parts - the client simply fetches them from the underlying `.filez` +object. diff --git a/docs/manual/introduction.md b/docs/manual/introduction.md new file mode 100644 index 00000000..b0d47390 --- /dev/null +++ b/docs/manual/introduction.md @@ -0,0 +1,110 @@ +# OSTree Overview + +## Introduction + +OSTree an upgrade system for Linux-based operating systems that +performs atomic upgrades of complete filesystem trees. It is +not a package system; rather, it is intended to complement them. +A primary model is composing packages on a server, and then +replicating them to clients. + +The underlying architecture might be summarized as "git for +operating system binaries". It operates in userspace, and will +work on top of any Linux filesystem. At its core is a git-like +content-addressed object store, and layered on top of that is +bootloader configuration, management of +`/etc`, and other functions to perform an +upgrade beyond just replicating files. + +You can use OSTree standalone in the pure replication model, +but another approach is to add a package manager on top, +thus creating a hybrid tree/package system. + +## Comparison with "package managers" + +Because OSTree is designed for deploying core operating +systems, a comparison with traditional "package managers" such +as dpkg and rpm is illustrative. Packages are traditionally +composed of partial filesystem trees with metadata and scripts +attached, and these are dynamically assembled on the client +machine, after a process of dependency resolution. + +In contrast, OSTree only supports recording and deploying +*complete* (bootable) filesystem trees. It +has no built-in knowledge of how a given filesystem tree was +generated or the origin of individual files, or dependencies, +descriptions of individual components. Put another way, OSTree +only handles delivery and deployment; you will likely still want +to include inside each tree metadata about the individual +components that went into the tree. For example, a system +administrator may want to know what version of OpenSSL was +included in your tree, so you should support the equivalent of +`rpm -q` or `dpkg -L`. + +The OSTree core emphasizes replicating read-only OS trees via +HTTP, and where the OS includes (if desired) an entirely +separate mechanism to install applications, stored in `/var` if they're system global, or +`/home` for per-user +application installation. An example application mechanism is +http://docker.io/ + +However, it is entirely possible to use OSTree underneath a +package system, where the contents of `/usr` are computed on the client. +For example, when installing a package, rather than changing the +currently running filesystem, the package manager could assemble +a new filesystem tree that layers the new packages on top of a +base tree, record it in the local OSTree repository, and then +set it up for the next boot. To support this model, OSTree +provides an (introspectable) C shared library. + +## Comparison with block/image replication + +OSTree shares some similarity with "dumb" replication and +stateless deployments, such as the model common in "cloud" +deployments where nodes are booted from an (effectively) +readonly disk, and user data is kept on a different volumes. +The advantage of "dumb" replication, shared by both OSTree and +the cloud model, is that it's *reliable* +and *predictable*. + +But unlike many default image-based deployments, OSTree supports +exactly two persistent writable directories that are preserved across +upgrades: `/etc` and `/var`. + +Because OSTree operates at the Unix filesystem layer, it works +on top of any filesystem or block storage layout; it's possible +to replicate a given filesystem tree from an OSTree repository +into plain ext4, BTRFS, XFS, or in general any Unix-compatible +filesystem that supports hard links. Note: OSTree will +transparently take advantage of some BTRFS features if deployed +on it. + +## Atomic transitions between parallel-installable read-only filesystem trees + +Another deeply fundamental difference between both package +managers and image-based replication is that OSTree is +designed to parallel-install *multiple versions* of multiple +*independent* operating systems. OSTree +relies on a new toplevel `ostree` directory; it can in fact +parallel install inside an existing OS or distribution +occupying the physical `/` root. + +On each client machine, there is an OSTree repository stored +in `/ostree/repo`, and a set of "deployments" stored in `/ostree/deploy/$OSNAME/$CHECKSUM`. +Each deployment is primarily composed of a set of hardlinks +into the repository. This means each version is deduplicated; +an upgrade process only costs disk space proportional to the +new files, plus some constant overhead. + +The model OSTree emphasizes is that the OS read-only content +is kept in the classic Unix `/usr`; it comes with code to +create a Linux read-only bind mount to prevent inadvertent +corruption. There is exactly one `/var` writable directory shared +between each deployment for a given OS. The OSTree core code +does not touch content in this directory; it is up to the code +in each operating system for how to manage and upgrade state. + +Finally, each deployment has its own writable copy of the +configuration store `/etc`. On upgrade, OSTree will +perform a basic 3-way diff, and apply any local changes to the +new copy, while leaving the old untouched. diff --git a/docs/manual/repo.md b/docs/manual/repo.md new file mode 100644 index 00000000..3b7a737f --- /dev/null +++ b/docs/manual/repo.md @@ -0,0 +1,90 @@ +# Anatomy of an OSTree repository + +## Core object types and data model + +OSTree is deeply inspired by git; the core layer is a userspace +content-addressed versioning filesystem. It is worth taking some time +to familiarize yourself with +[Git Internals](http://git-scm.com/book/en/Git-Internals), as this +section will assume some knowledge of how git works. + +Its object types are similar to git; it has commit objects and content +objects. Git has "tree" objects, whereas OSTree splits them into +"dirtree" and "dirmeta" objects. But unlike git, OSTree's checksums +are SHA256. And most crucially, its content objects include uid, gid, +and extended attributes (but still no timestamps). + +### Commit objects + +A commit object contains metadata such as a timestamp, a log +message, and most importantly, a reference to a +dirtree/dirmeta pair of checksums which describe the root +directory of the filesystem. +Also like git, each commit in OSTree can have a parent. It is +designed to store a history of your binary builds, just like git +stores a history of source control. However, OSTree also makes +it easy to delete data, under the assumption that you can +regenerate it from source code. + +### Dirtree objects + +A dirtree contains a sorted array of (filename, checksum) +pairs for content objects, and a second sorted array of +(filename, dirtree checksum, dirmeta checksum), which are +subdirectories. + +### Dirmeta objects + +In git, tree objects contain the metadata such as permissions +for their children. But OSTree splits this into a separate +object to avoid duplicating extended attribute listings. + +### Content objects + +Unlike the first three object types which are metadata, designed to be +`mmap()`ed, the content object has a separate internal header and +payload sections. The header contains uid, gid, mode, and symbolic +link target (for symlinks), as well as extended attributes. After the +header, for regular files, the content follows. + +# Repository types and locations + +Also unlike git, an OSTree repository can be in one of three separate +modes: `bare`, `bare-user`, and `archive-z2`. A bare repository is +one where content files are just stored as regular files; it's +designed to be the source of a "hardlink farm", where each operating +system checkout is merely links into it. If you want to store files +owned by e.g. root in this mode, you must run OSTree as root. + +The `bare-user` is a later addition that is like `bare` in that files +are unpacked, but it can (and should generally) be created as +non-root. In this mode, extended metadata such as owner uid, gid, and +extended attributes are stored but not actually applied. +The `bare-user` mode is useful for build systems that run as non-root +but want to generate root-owned content, as well as non-root container +systems. + +In contrast, the `archive-z2` mode is designed for serving via plain +HTTP. Like tar files, it can be read/written by non-root users. + +On an OSTree-deployed system, the "system repository" is +`/ostree/repo`. It can be read by any uid, but only written by root. +Unless the `--repo` argument is given to the ostree +command, it will operate on the system repository. + +## Refs + +Like git, OSTree uses "refs" to which are text files that point to +particular commits (i.e. filesystem trees). For example, the +gnome-ostree operating system creates trees named like +`exampleos/buildmaster/x86_64-runtime` and +`exampleos/buildmaster/x86_64-devel-debug`. These two refs point to +two different generated filesystem trees. In this example, the +"runtime" tree contains just enough to run a basic system, and +"devel-debug" contains all of the developer tools and debuginfo. + +The `ostree` supports a simple syntax using the carat `^` to refer to +the parent of a given commit. For example, +`exampleos/buildmaster/x86_64-runtime^` refers to the previous build, +and `exampleos/buildmaster/x86_64-runtime^^` refers to the one before +that. diff --git a/libglnx/Makefile-libglnx.am b/libglnx/Makefile-libglnx.am index 712e931c..2e8da015 100644 --- a/libglnx/Makefile-libglnx.am +++ b/libglnx/Makefile-libglnx.am @@ -18,6 +18,7 @@ EXTRA_DIST += $(libglnx_srcpath)/README.md $(libglnx_srcpath)/COPYING libglnx_la_SOURCES = \ + $(libglnx_srcpath)/glnx-alloca.h \ $(libglnx_srcpath)/glnx-backport-autocleanups.h \ $(libglnx_srcpath)/glnx-backport-autoptr.h \ $(libglnx_srcpath)/glnx-backports.h \ diff --git a/libglnx/glnx-alloca.h b/libglnx/glnx-alloca.h new file mode 100644 index 00000000..b0bf6a65 --- /dev/null +++ b/libglnx/glnx-alloca.h @@ -0,0 +1,47 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2014,2015 Colin Walters . + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#pragma once + +#include +#include + +G_BEGIN_DECLS + +/* Taken from https://github.com/systemd/systemd/src/basic/string-util.h + * at revision v228-666-gcf6c8c4 + */ +#define glnx_strjoina(a, ...) \ + ({ \ + const char *_appendees_[] = { a, __VA_ARGS__ }; \ + char *_d_, *_p_; \ + int _len_ = 0; \ + unsigned _i_; \ + for (_i_ = 0; _i_ < G_N_ELEMENTS(_appendees_) && _appendees_[_i_]; _i_++) \ + _len_ += strlen(_appendees_[_i_]); \ + _p_ = _d_ = alloca(_len_ + 1); \ + for (_i_ = 0; _i_ < G_N_ELEMENTS(_appendees_) && _appendees_[_i_]; _i_++) \ + _p_ = stpcpy(_p_, _appendees_[_i_]); \ + *_p_ = 0; \ + _d_; \ + }) + + +G_END_DECLS diff --git a/libglnx/glnx-backport-autocleanups.h b/libglnx/glnx-backport-autocleanups.h index 53f7ffd2..cc249611 100644 --- a/libglnx/glnx-backport-autocleanups.h +++ b/libglnx/glnx-backport-autocleanups.h @@ -31,13 +31,6 @@ g_autoptr_cleanup_generic_gfree (void *p) g_free (*pp); } -static inline void -g_autoptr_cleanup_gstring_free (GString *string) -{ - if (string) - g_string_free (string, TRUE); -} - G_DEFINE_AUTOPTR_CLEANUP_FUNC(GAsyncQueue, g_async_queue_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GBookmarkFile, g_bookmark_file_free) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GBytes, g_bytes_unref) @@ -55,7 +48,6 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(GPtrArray, g_ptr_array_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMainContext, g_main_context_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMainLoop, g_main_loop_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GSource, g_source_unref) -G_DEFINE_AUTOPTR_CLEANUP_FUNC(GString, g_autoptr_cleanup_gstring_free) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMappedFile, g_mapped_file_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GMarkupParseContext, g_markup_parse_context_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(gchar, g_free) @@ -113,5 +105,20 @@ G_DEFINE_AUTOPTR_CLEANUP_FUNC(GTlsDatabase, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GTlsInteraction, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GDBusConnection, g_object_unref) G_DEFINE_AUTOPTR_CLEANUP_FUNC(GDBusMessage, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GZlibCompressor, g_object_unref) +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GZlibDecompressor, g_object_unref) + +#endif + +#if !GLIB_CHECK_VERSION(2, 45, 8) + +static inline void +g_autoptr_cleanup_gstring_free (GString *string) +{ + if (string) + g_string_free (string, TRUE); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(GString, g_autoptr_cleanup_gstring_free) #endif diff --git a/libglnx/glnx-console.c b/libglnx/glnx-console.c index deb4c86c..39733118 100644 --- a/libglnx/glnx-console.c +++ b/libglnx/glnx-console.c @@ -241,7 +241,7 @@ glnx_console_progress_text_percent (const char *text, if (textlen > 0) { - fwrite (text, 1, textlen - 1, stdout); + fwrite (text, 1, textlen, stdout); fputc (' ', stdout); } @@ -285,5 +285,5 @@ glnx_console_unlock (GLnxConsoleRef *console) if (console->is_tty) fputc ('\n', stdout); - locked = FALSE; + locked = console->locked = FALSE; } diff --git a/libglnx/glnx-console.h b/libglnx/glnx-console.h index 9f620cc7..8fc38656 100644 --- a/libglnx/glnx-console.h +++ b/libglnx/glnx-console.h @@ -45,7 +45,8 @@ guint glnx_console_columns (void); static inline void glnx_console_ref_cleanup (GLnxConsoleRef *p) { - glnx_console_unlock (p); + if (p->locked) + glnx_console_unlock (p); } G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(GLnxConsoleRef, glnx_console_ref_cleanup) diff --git a/libglnx/glnx-dirfd.c b/libglnx/glnx-dirfd.c index d126f15a..4861ccfe 100644 --- a/libglnx/glnx-dirfd.c +++ b/libglnx/glnx-dirfd.c @@ -168,6 +168,8 @@ glnx_dirfd_iterator_next_dent (GLnxDirFdIterator *dfd_iter, gboolean ret = FALSE; GLnxRealDirfdIterator *real_dfd_iter = (GLnxRealDirfdIterator*) dfd_iter; + g_return_val_if_fail (out_dent, FALSE); + if (g_cancellable_set_error_if_cancelled (cancellable, error)) goto out; @@ -189,6 +191,53 @@ glnx_dirfd_iterator_next_dent (GLnxDirFdIterator *dfd_iter, return ret; } +/** + * glnx_dirfd_iterator_next_dent_ensure_dtype: + * @dfd_iter: A directory iterator + * @out_dent: (out) (transfer none): Pointer to dirent; do not free + * @cancellable: Cancellable + * @error: Error + * + * A variant of @glnx_dirfd_iterator_next_dent, which will ensure the + * `dent->d_type` member is filled in by calling `fstatat` + * automatically if the underlying filesystem type sets `DT_UNKNOWN`. + */ +gboolean +glnx_dirfd_iterator_next_dent_ensure_dtype (GLnxDirFdIterator *dfd_iter, + struct dirent **out_dent, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + struct dirent *ret_dent; + + g_return_val_if_fail (out_dent, FALSE); + + if (!glnx_dirfd_iterator_next_dent (dfd_iter, out_dent, cancellable, error)) + goto out; + + ret_dent = *out_dent; + + if (ret_dent) + { + + if (ret_dent->d_type == DT_UNKNOWN) + { + struct stat stbuf; + if (TEMP_FAILURE_RETRY (fstatat (dfd_iter->fd, ret_dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW)) != 0) + { + glnx_set_error_from_errno (error); + goto out; + } + ret_dent->d_type = IFTODT (stbuf.st_mode); + } + } + + ret = TRUE; + out: + return ret; +} + /** * glnx_dirfd_iterator_clear: * @dfd_iter: Iterator, will be de-initialized diff --git a/libglnx/glnx-dirfd.h b/libglnx/glnx-dirfd.h index 5b7b77a9..c13e3dc5 100644 --- a/libglnx/glnx-dirfd.h +++ b/libglnx/glnx-dirfd.h @@ -60,6 +60,10 @@ gboolean glnx_dirfd_iterator_next_dent (GLnxDirFdIterator *dfd_iter, struct dirent **out_dent, GCancellable *cancellable, GError **error); +gboolean glnx_dirfd_iterator_next_dent_ensure_dtype (GLnxDirFdIterator *dfd_iter, + struct dirent **out_dent, + GCancellable *cancellable, + GError **error); void glnx_dirfd_iterator_clear (GLnxDirFdIterator *dfd_iter); G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(GLnxDirFdIterator, glnx_dirfd_iterator_clear) diff --git a/libglnx/glnx-fdio.c b/libglnx/glnx-fdio.c index 7db33c4c..466cbc4e 100644 --- a/libglnx/glnx-fdio.c +++ b/libglnx/glnx-fdio.c @@ -354,7 +354,7 @@ static int btrfs_reflink(int infd, int outfd) { return 0; } -static int loop_write(int fd, const void *buf, size_t nbytes) { +int glnx_loop_write(int fd, const void *buf, size_t nbytes) { const uint8_t *p = buf; g_return_val_if_fail(fd >= 0, -1); @@ -437,7 +437,7 @@ static int copy_bytes(int fdf, int fdt, off_t max_bytes, bool try_reflink) { if (n == 0) /* EOF */ break; - r = loop_write(fdt, buf, (size_t) n); + r = glnx_loop_write(fdt, buf, (size_t) n); if (r < 0) return r; } @@ -685,7 +685,7 @@ glnx_file_replace_contents_with_perms_at (int dfd, goto out; } - if ((r = loop_write (fd, buf, len)) != 0) + if ((r = glnx_loop_write (fd, buf, len)) != 0) { errno = -r; glnx_set_error_from_errno (error); diff --git a/libglnx/glnx-fdio.h b/libglnx/glnx-fdio.h index a90544a9..c0fd4e48 100644 --- a/libglnx/glnx-fdio.h +++ b/libglnx/glnx-fdio.h @@ -102,6 +102,9 @@ glnx_readlinkat_malloc (int dfd, GCancellable *cancellable, GError **error); +int +glnx_loop_write (int fd, const void *buf, size_t nbytes); + typedef enum { GLNX_FILE_COPY_OVERWRITE, GLNX_FILE_COPY_NOXATTRS, diff --git a/libglnx/libglnx.h b/libglnx/libglnx.h index 9bd41c45..451740d2 100644 --- a/libglnx/libglnx.h +++ b/libglnx/libglnx.h @@ -24,6 +24,7 @@ G_BEGIN_DECLS +#include #include #include #include diff --git a/doc/ostree-admin-cleanup.xml b/man/ostree-admin-cleanup.xml similarity index 100% rename from doc/ostree-admin-cleanup.xml rename to man/ostree-admin-cleanup.xml diff --git a/doc/ostree-admin-config-diff.xml b/man/ostree-admin-config-diff.xml similarity index 100% rename from doc/ostree-admin-config-diff.xml rename to man/ostree-admin-config-diff.xml diff --git a/doc/ostree-admin-deploy.xml b/man/ostree-admin-deploy.xml similarity index 100% rename from doc/ostree-admin-deploy.xml rename to man/ostree-admin-deploy.xml diff --git a/doc/ostree-admin-init-fs.xml b/man/ostree-admin-init-fs.xml similarity index 100% rename from doc/ostree-admin-init-fs.xml rename to man/ostree-admin-init-fs.xml diff --git a/doc/ostree-admin-instutil.xml b/man/ostree-admin-instutil.xml similarity index 100% rename from doc/ostree-admin-instutil.xml rename to man/ostree-admin-instutil.xml diff --git a/doc/ostree-admin-os-init.xml b/man/ostree-admin-os-init.xml similarity index 100% rename from doc/ostree-admin-os-init.xml rename to man/ostree-admin-os-init.xml diff --git a/doc/ostree-admin-set-origin.xml b/man/ostree-admin-set-origin.xml similarity index 100% rename from doc/ostree-admin-set-origin.xml rename to man/ostree-admin-set-origin.xml diff --git a/doc/ostree-admin-status.xml b/man/ostree-admin-status.xml similarity index 100% rename from doc/ostree-admin-status.xml rename to man/ostree-admin-status.xml diff --git a/doc/ostree-admin-switch.xml b/man/ostree-admin-switch.xml similarity index 100% rename from doc/ostree-admin-switch.xml rename to man/ostree-admin-switch.xml diff --git a/doc/ostree-admin-undeploy.xml b/man/ostree-admin-undeploy.xml similarity index 100% rename from doc/ostree-admin-undeploy.xml rename to man/ostree-admin-undeploy.xml diff --git a/doc/ostree-admin-upgrade.xml b/man/ostree-admin-upgrade.xml similarity index 100% rename from doc/ostree-admin-upgrade.xml rename to man/ostree-admin-upgrade.xml diff --git a/doc/ostree-admin.xml b/man/ostree-admin.xml similarity index 100% rename from doc/ostree-admin.xml rename to man/ostree-admin.xml diff --git a/doc/ostree-cat.xml b/man/ostree-cat.xml similarity index 100% rename from doc/ostree-cat.xml rename to man/ostree-cat.xml diff --git a/doc/ostree-checkout.xml b/man/ostree-checkout.xml similarity index 100% rename from doc/ostree-checkout.xml rename to man/ostree-checkout.xml diff --git a/doc/ostree-checksum.xml b/man/ostree-checksum.xml similarity index 100% rename from doc/ostree-checksum.xml rename to man/ostree-checksum.xml diff --git a/doc/ostree-commit.xml b/man/ostree-commit.xml similarity index 100% rename from doc/ostree-commit.xml rename to man/ostree-commit.xml diff --git a/doc/ostree-config.xml b/man/ostree-config.xml similarity index 100% rename from doc/ostree-config.xml rename to man/ostree-config.xml diff --git a/doc/ostree-diff.xml b/man/ostree-diff.xml similarity index 100% rename from doc/ostree-diff.xml rename to man/ostree-diff.xml diff --git a/man/ostree-export.xml b/man/ostree-export.xml new file mode 100644 index 00000000..39ad8e69 --- /dev/null +++ b/man/ostree-export.xml @@ -0,0 +1,70 @@ + + + + + + + + + ostree export + OSTree + + + + Developer + Colin + Walters + walters@verbum.org + + + + + + ostree export + 1 + + + + ostree-export + Generate a tar archive from an OSTree commit + + + + + ostree export OPTIONS BRANCH + + + + + Description + + + This command generates a GNU tar formatted archive from an + OSTree commit. This is useful for cases like backups, + converting OSTree commits into Docker images, and the like. + + + + + Example + $ ostree export exampleos/x86_64/standard | gzip > exampleos-standard.tar.gz + + diff --git a/doc/ostree-fsck.xml b/man/ostree-fsck.xml similarity index 100% rename from doc/ostree-fsck.xml rename to man/ostree-fsck.xml diff --git a/doc/ostree-gpg-sign.xml b/man/ostree-gpg-sign.xml similarity index 100% rename from doc/ostree-gpg-sign.xml rename to man/ostree-gpg-sign.xml diff --git a/doc/ostree-init.xml b/man/ostree-init.xml similarity index 100% rename from doc/ostree-init.xml rename to man/ostree-init.xml diff --git a/doc/ostree-log.xml b/man/ostree-log.xml similarity index 100% rename from doc/ostree-log.xml rename to man/ostree-log.xml diff --git a/doc/ostree-ls.xml b/man/ostree-ls.xml similarity index 100% rename from doc/ostree-ls.xml rename to man/ostree-ls.xml diff --git a/doc/ostree-prune.xml b/man/ostree-prune.xml similarity index 100% rename from doc/ostree-prune.xml rename to man/ostree-prune.xml diff --git a/doc/ostree-pull-local.xml b/man/ostree-pull-local.xml similarity index 100% rename from doc/ostree-pull-local.xml rename to man/ostree-pull-local.xml diff --git a/doc/ostree-pull.xml b/man/ostree-pull.xml similarity index 75% rename from doc/ostree-pull.xml rename to man/ostree-pull.xml index d915bcd2..c419307e 100644 --- a/doc/ostree-pull.xml +++ b/man/ostree-pull.xml @@ -111,14 +111,37 @@ Boston, MA 02111-1307, USA. Description - Downloads all content corresponding to the provided branch or commit from the given remote. + This command can retrieve just a specific commit, or go all + the way to performing a full mirror of the remote + repository. If no BRANCH is specified, + all branches are retrieved. + + + A special syntax in the @ character allows + specifying a specific commit to retrieve from a branch. This + + Example - $ ostree pull remote_name + $ ostree --repo=repo pull --depth=-1 --mirror remote_name + + Perform a complete mirror of the remote. (This is + likely most useful if your repository is also + archive-z2 mode) + + $ ostree --repo=repo pull remote_name exampleos/x86_64/standard + + Fetch the most recent commit to exampleos/x86_64/standard. + + $ ostree --repo=repo pull remote_name exampleos/x86_64/standard@98ea6e4f216f2fb4b69fff9b3a44842c38686ca685f3f55dc48c5d3fb1107be4 + + Download the specific commit starting with + 98ea6e as if it was the latest commit for + exampleos/x86_64/standard. diff --git a/doc/ostree-refs.xml b/man/ostree-refs.xml similarity index 100% rename from doc/ostree-refs.xml rename to man/ostree-refs.xml diff --git a/doc/ostree-remote.xml b/man/ostree-remote.xml similarity index 100% rename from doc/ostree-remote.xml rename to man/ostree-remote.xml diff --git a/doc/ostree-reset.xml b/man/ostree-reset.xml similarity index 100% rename from doc/ostree-reset.xml rename to man/ostree-reset.xml diff --git a/doc/ostree-rev-parse.xml b/man/ostree-rev-parse.xml similarity index 100% rename from doc/ostree-rev-parse.xml rename to man/ostree-rev-parse.xml diff --git a/doc/ostree-show.xml b/man/ostree-show.xml similarity index 100% rename from doc/ostree-show.xml rename to man/ostree-show.xml diff --git a/doc/ostree-static-delta.xml b/man/ostree-static-delta.xml similarity index 100% rename from doc/ostree-static-delta.xml rename to man/ostree-static-delta.xml diff --git a/doc/ostree-summary.xml b/man/ostree-summary.xml similarity index 100% rename from doc/ostree-summary.xml rename to man/ostree-summary.xml diff --git a/doc/ostree-trivial-httpd.xml b/man/ostree-trivial-httpd.xml similarity index 100% rename from doc/ostree-trivial-httpd.xml rename to man/ostree-trivial-httpd.xml diff --git a/doc/ostree.repo-config.xml b/man/ostree.repo-config.xml similarity index 100% rename from doc/ostree.repo-config.xml rename to man/ostree.repo-config.xml diff --git a/doc/ostree.repo.xml b/man/ostree.repo.xml similarity index 100% rename from doc/ostree.repo.xml rename to man/ostree.repo.xml diff --git a/doc/ostree.xml b/man/ostree.xml similarity index 100% rename from doc/ostree.xml rename to man/ostree.xml diff --git a/man/rofiles-fuse.xml b/man/rofiles-fuse.xml new file mode 100644 index 00000000..3d282c00 --- /dev/null +++ b/man/rofiles-fuse.xml @@ -0,0 +1,104 @@ + + + + + + + + + rofiles-fuse + rofiles-fuse + + + + Developer + Colin + Walters + walters@verbum.org + + + + + + rofiles-fuse + 1 + + + + rofiles-fuse + Use FUSE to create a view where directories are writable, files are immutable + + + + + rofiles-fuse SRCDIR MNTPOINT + + + + + Description + + + Creating a checkout from an OSTree repository by default + uses hard links, which means an in-place mutation to any + file corrupts the repository and all checkouts. This can be + problematic if one wishes to run arbitrary programs against + such a checkout. For example, RPM %post + scripts or equivalent. + + + + In the case where one wants to create a tree commit derived + from other content, using rofiles-fuse in + concert with ostree commit + --link-checkout-speedup (or the underlying API) + can ensure that only new files are checksummed. + + + + + + Example: Update an OSTree commit + +# Initialize a checkout and mount +$ ostree --repo=repo checkout somebranch branch-checkout +$ mkdir mnt +$ rofiles-fuse branch-checkout mnt + +# Now, arbitrary changes to mnt/ are reflected in branch-checkout +$ echo somenewcontent > mnt/anewfile +$ mkdir mnt/anewdir +$ rm mnt/someoriginalcontent -rf + +# Commit and cleanup +$ fusermount -u mnt +$ ostree --repo=repo commit --link-checkout-speedup -b somebranch -s 'Commit new content' --tree=dir=branch-checkout +$ rm mnt branch-checkout -rf + + + + + See Also + + ostree1 + + + diff --git a/manual-tests/static-delta-generate-crosscheck.sh b/manual-tests/static-delta-generate-crosscheck.sh new file mode 100755 index 00000000..4dbaab98 --- /dev/null +++ b/manual-tests/static-delta-generate-crosscheck.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Generate multiple variants of static deltas in a repo, then use them +# to upgrade in a test repository, verifying that fsck works. +# +# This test is manual so it's easy to run against arbitrary content. + +set -euo pipefail + +repo=$(pwd)/${1} +shift +branch=${1} +shift + +from=$(ostree --repo=${repo} rev-parse "${branch}"^) +to=$(ostree --repo=${repo} rev-parse ${branch}) + +tmpdir=$(mktemp -d /var/tmp/ostree-delta-check.XXXXXX) +cd ${tmpdir} +touch ${tmpdir}/.tmp +echo "Using tmpdir ${tmpdir}" + +cleanup_tmpdir() { + if test -f ${tmpdir}/.tmp; then + rm -rf ${tmpdir} + fi +} + +if test -z "${PRESERVE_TMP:-}"; then + trap cleanup_tmpdir EXIT +fi + +fatal() { + echo "$@" + exit 1 +} + +assert_streq() { + if ! test $1 = $2; then + fatal "assertion failed: $1 = $2" + fi +} + +validate_delta_options() { + mkdir testrepo + ostree --repo=testrepo init --mode=bare-user + ostree --repo=testrepo remote add --set=gpg-verify=false local file://${repo} + ostree --repo=${repo} static-delta generate $@ ${from} ${to} + ostree --repo=testrepo pull local ${branch}@${from} + assert_streq $(ostree --repo=testrepo rev-parse ${branch}) ${from} + ostree --repo=testrepo pull local ${branch} + assert_streq $(ostree --repo=testrepo rev-parse ${branch}) ${to} + ostree --repo=testrepo fsck + rm testrepo -rf +} + +set -x + +validate_delta_options +validate_delta_options --inline +validate_delta_options --disable-bsdiff + diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..89211c79 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,11 @@ +site_name: My Docs +pages: + - Home: 'index.md' + - Contributing: 'CONTRIBUTING.md' + - Manual: + - Introduction: 'manual/introduction.md' + - Repository: 'manual/repo.md' + - Deployments: 'manual/deployment.md' + - Atomic Upgrades: 'manual/atomic-upgrades.md' + - Adapting Existing Systems: 'manual/adapting-existing.md' + - Formats: 'manual/formats.md' diff --git a/packaging/ostree.spec.in b/packaging/ostree.spec.in index 8fb219d6..2d6a2660 100644 --- a/packaging/ostree.spec.in +++ b/packaging/ostree.spec.in @@ -19,11 +19,14 @@ BuildRequires: pkgconfig(libgsystem) BuildRequires: pkgconfig(e2p) # Extras BuildRequires: pkgconfig(libarchive) +BuildRequires: pkgconfig(fuse) BuildRequires: pkgconfig(libselinux) +BuildRequires: libcap-devel BuildRequires: gpgme-devel BuildRequires: pkgconfig(systemd) BuildRequires: /usr/bin/g-ir-scanner BuildRequires: dracut +BuildRequires: bison # Runtime requirements Requires: dracut @@ -53,6 +56,14 @@ Requires: grub2 %description grub2 GRUB2 integration for OSTree +%package fuse +Summary: FUSE utilities for OSTree +Group: Development/Libraries +Requires: fuse + +%description fuse +%{summary} + %prep %setup -q -n ostree-%{version} @@ -106,3 +117,6 @@ rm -rf $RPM_BUILD_ROOT %files grub2 %{_sysconfdir}/grub.d/*ostree %{_libexecdir}/ostree/grub2* + +%files fuse +%{_bindir}/rofiles-fuse diff --git a/src/libostree/README-deltas.md b/src/libostree/README-deltas.md deleted file mode 100644 index 9e09e45a..00000000 --- a/src/libostree/README-deltas.md +++ /dev/null @@ -1,158 +0,0 @@ -OSTree Static Object Deltas -=========================== - -Currently, OSTree's "archive-z2" mode stores both metadata and content -objects as individual files in the filesystem. Content objects are -zlib-compressed. - -The advantage of this is model are: - -0) It's easy to understand and implement -1) Can be served directly over plain HTTP by a static webserver -2) Space efficient on the server - -However, it can be inefficient both for large updates and small ones: - -0) For large tree changes (such as going from -runtime to - -devel-debug, or major version upgrades), this can mean thousands - and thousands of HTTP requests. The overhead for that is very - large (until SPDY/HTTP2.0), and will be catastrophically bad if the - webserver is not configured with KeepAlive. -1) Small changes (typo in gnome-shell .js file) still require around - 5 metadata HTTP requests, plus a redownload of the whole file. - -Why not smart servers? -====================== - -Smart servers (custom daemons, or just CGI scripts) as git has are not -under consideration for this proposal. OSTree is designed for the -same use case as GNU/Linux distribution package systems are, where -content is served by a network of volunteer mirrors that will -generally not run custom code. - -In particular, Amazon S3 style dumb content servers is a very -important use case, as is being able to apply updates from static -media like DVD-ROM. - -Finding Static Deltas -===================== - -Since static deltas may not exist, the client first needs to attempt -to locate one. Suppose a client wants to retrieve commit ${new} while -currently running ${current}. The first thing to fetch is the delta -metadata, called "meta". It can be found at -${repo}/deltas/${current}-${new}/meta. - -FIXME: GPG signatures (.metameta?) Or include commit object in meta? -But we would then be forced to verify the commit only after processing -the entirety of the delta, which is dangerous. I think we need to -require signing deltas. - -Delta Bytecode Format -===================== - -A delta-part has the following form: - -byte compression-type (0 = none, 'g' = gzip') -REPEAT[(varint size, delta-part-content)] - -delta-part-content: - byte[] payload - ARRAY[operation] - -The rationale for having delta-part is that it allows easy incremental -resumption of downloads. The client can look at the delta descriptor -and skip downloading delta-parts for which it already has the -contained objects. This is better than simply resuming a gigantic -file because if the client decides to fetch a slightly newer version, -it's very probable that some of the downloading we've already done is -still useful. - -For the actual delta payload, it comes as a stream of pair of -(payload, operation) so that it can be processed while being -decompressed. - -Finally, the delta-part-content is effectively a high level bytecode -for a stack-oriented machine. It iterates on the array of objects in -order. The following operations are available: - -FETCH - Fall back to fetching the current object individually. Move - to the next object. - -WRITE(array[(varint offset, varint length)]) - Write from current input target (default payload) to output. - -GUNZIP(array[(varint offset, varint length)]) - gunzip from current input target (default payload) to output. - -CLOSE - Close the current output target, and proceed to the next; if the - output object was a temporary, the output resets to the current - object. - -# Change the input source to an object -READOBJECT(csum object) - Set object as current input target - -# Change the input source to payload -READPAYLOAD - Set payload as current input target - -Compiling Deltas -================ - -After reading the above, you may be wondering how we actually *make* -these deltas. I envison a strategy similar to that employed by -Chromium autoupdate: -http://www.chromium.org/chromium-os/chromiumos-design-docs/autoupdate-details - -Something like this would be a useful initial algorithm: -1) Compute the set of added objects NEW -2) For each object in NEW: - - Look for a the set of "superficially similar" objects in the - previous tree, using heuristics based first on filename (including - prefix), then on size. Call this set CANDIDATES. - For each entry in CANDIDATES: - - Try doing a bup/librsync style rolling checksum, and compute the - list of changed blocks. - - Try gzip-compressing it -3) Choose the lowest cost method for each NEW object, and partition - the program for each method into deltapart-sized chunks. - -However, there are many other possibilities, that could be used in a -hybrid mode with the above. For example, we could try to find similar -objects, and gzip them together. This would be a *very* useful -strategy for things like the 9000 Boost headers which have massive -amounts of redundant data. - -Notice too that the delta format supports falling back to retrieving -individual objects. For cases like the initramfs which is compressed -inside the tree with gzip, we're not going to find an efficient way to -sync it, so the delta compiler should just fall back to fetching it -individually. - -Which Deltas To Create? -======================= - -Going back to the start, there are two cases to optimize for: - -1) Incremental upgrades between builds -2) Major version upgrades - -A command line operation would look something like this: - -$ ostree --repo=/path/to/repo gendelta --ref-prefix=gnome-ostree/buildmaster/ --strategy=latest --depth=5 - -This would tell ostree to generate deltas from each of the last 4 -commits to each ref (e.g. gnome-ostree/buildmaster/x86_64-runtime) to -the latest commit. It might also be possible of course to have ---strategy=incremental where we generate a delta between each commit. -I suspect that'd be something to do if one has a *lot* of disk space -to spend, and there's a reason for clients to be fetching individual -refs. - -$ ostree --repo=/path/to/repo gendelta --from=gnome-ostree/3.10/x86_64-runtime --to=gnome-ostree/buildmaster/x86_64-runtime - -This is an obvious one - generate a delta from the last stable release -to the current development head. diff --git a/src/libostree/ostree-bootloader-grub2.c b/src/libostree/ostree-bootloader-grub2.c index 1f899143..42ed5acc 100644 --- a/src/libostree/ostree-bootloader-grub2.c +++ b/src/libostree/ostree-bootloader-grub2.c @@ -388,8 +388,6 @@ rm -f ${grub_cfg}.new if (!g_file_copy (self->config_path_efi, config_path_efi_old, G_FILE_COPY_OVERWRITE, cancellable, NULL, NULL, error)) goto out; - if (!ot_gfile_ensure_unlinked (config_path_efi_old, cancellable, error)) - goto out; /* NOTE: NON-ATOMIC REPLACEMENT; WE can't do anything else on FAT; * see https://bugzilla.gnome.org/show_bug.cgi?id=724246 diff --git a/src/libostree/ostree-cmdprivate.c b/src/libostree/ostree-cmdprivate.c index 99834937..74c32a3c 100644 --- a/src/libostree/ostree-cmdprivate.c +++ b/src/libostree/ostree-cmdprivate.c @@ -23,6 +23,7 @@ #include "ostree-cmdprivate.h" #include "ostree-repo-private.h" #include "ostree-core-private.h" +#include "ostree-repo-static-delta-private.h" #include "ostree-sysroot.h" #include "ostree-bootloader-grub2.h" @@ -44,7 +45,8 @@ const OstreeCmdPrivateVTable * ostree_cmd__private__ (void) { static OstreeCmdPrivateVTable table = { - impl_ostree_generate_grub2_config + impl_ostree_generate_grub2_config, + _ostree_repo_static_delta_dump }; return &table; diff --git a/src/libostree/ostree-cmdprivate.h b/src/libostree/ostree-cmdprivate.h index 317a7592..7746406b 100644 --- a/src/libostree/ostree-cmdprivate.h +++ b/src/libostree/ostree-cmdprivate.h @@ -26,6 +26,7 @@ G_BEGIN_DECLS typedef struct { gboolean (* ostree_generate_grub2_config) (OstreeSysroot *sysroot, int bootversion, int target_fd, GCancellable *cancellable, GError **error); + gboolean (* ostree_static_delta_dump) (OstreeRepo *repo, const char *delta_id, GCancellable *cancellable, GError **error); } OstreeCmdPrivateVTable; const OstreeCmdPrivateVTable * diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index 5f24e7c9..11c61f95 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -1069,7 +1069,7 @@ int ostree_cmp_checksum_bytes (const guchar *a, const guchar *b) { - return memcmp (a, b, 32); + return memcmp (a, b, OSTREE_SHA256_DIGEST_LEN); } /** @@ -1151,7 +1151,7 @@ ostree_checksum_inplace_to_bytes (const char *checksum, guint i; guint j; - for (i = 0, j = 0; i < 32; i += 1, j += 2) + for (i = 0, j = 0; i < OSTREE_SHA256_DIGEST_LEN; i += 1, j += 2) { gint big, little; @@ -1177,7 +1177,7 @@ ostree_checksum_inplace_to_bytes (const char *checksum, guchar * ostree_checksum_to_bytes (const char *checksum) { - guchar *ret = g_malloc (32); + guchar *ret = g_malloc (OSTREE_SHA256_DIGEST_LEN); ostree_checksum_inplace_to_bytes (checksum, ret); return ret; } @@ -1191,9 +1191,9 @@ ostree_checksum_to_bytes (const char *checksum) GVariant * ostree_checksum_to_bytes_v (const char *checksum) { - guchar result[32]; + guchar result[OSTREE_SHA256_DIGEST_LEN]; ostree_checksum_inplace_to_bytes (checksum, result); - return ot_gvariant_new_bytearray ((guchar*)result, 32); + return ot_gvariant_new_bytearray ((guchar*)result, OSTREE_SHA256_DIGEST_LEN); } /** @@ -1210,7 +1210,7 @@ ostree_checksum_inplace_from_bytes (const guchar *csum, static const gchar hexchars[] = "0123456789abcdef"; guint i, j; - for (i = 0, j = 0; i < 32; i++, j += 2) + for (i = 0, j = 0; i < OSTREE_SHA256_DIGEST_LEN; i++, j += 2) { guchar byte = csum[i]; buf[j] = hexchars[byte >> 4]; @@ -1242,7 +1242,7 @@ ostree_checksum_b64_inplace_from_bytes (const guchar *csum, * a lot easier to reuse GLib's base64 encoder and postprocess it * to replace the '/' with '_'. */ - outlen = g_base64_encode_step (csum, 32, FALSE, tmpbuf, &state, &save); + outlen = g_base64_encode_step (csum, OSTREE_SHA256_DIGEST_LEN, FALSE, tmpbuf, &state, &save); outlen += g_base64_encode_close (FALSE, tmpbuf+outlen, &state, &save); g_assert (outlen == 44); @@ -1299,7 +1299,7 @@ ostree_checksum_bytes_peek (GVariant *bytes) gsize n_elts; const guchar *ret; ret = g_variant_get_fixed_array (bytes, &n_elts, 1); - if (G_UNLIKELY (n_elts != 32)) + if (G_UNLIKELY (n_elts != OSTREE_SHA256_DIGEST_LEN)) return NULL; return ret; } @@ -1434,20 +1434,20 @@ _ostree_get_relative_static_delta_path (const char *from, const char *to, const char *target) { - guint8 csum_to[32]; + guint8 csum_to[OSTREE_SHA256_DIGEST_LEN]; char to_b64[44]; - guint8 csum_to_copy[32]; + guint8 csum_to_copy[OSTREE_SHA256_DIGEST_LEN]; GString *ret = g_string_new ("deltas/"); ostree_checksum_inplace_to_bytes (to, csum_to); ostree_checksum_b64_inplace_from_bytes (csum_to, to_b64); ostree_checksum_b64_inplace_to_bytes (to_b64, csum_to_copy); - g_assert (memcmp (csum_to, csum_to_copy, 32) == 0); + g_assert (memcmp (csum_to, csum_to_copy, OSTREE_SHA256_DIGEST_LEN) == 0); if (from != NULL) { - guint8 csum_from[32]; + guint8 csum_from[OSTREE_SHA256_DIGEST_LEN]; char from_b64[44]; ostree_checksum_inplace_to_bytes (from, csum_from); diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index 637f8172..c5b42a75 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -50,6 +50,8 @@ G_BEGIN_DECLS */ #define OSTREE_MAX_RECURSION (256) +#define OSTREE_SHA256_DIGEST_LEN (32) + /** * OstreeObjectType: * @OSTREE_OBJECT_TYPE_FILE: Content; regular file, symbolic link diff --git a/src/libostree/ostree-fetcher.c b/src/libostree/ostree-fetcher.c index 665286c1..b9223217 100644 --- a/src/libostree/ostree-fetcher.c +++ b/src/libostree/ostree-fetcher.c @@ -277,7 +277,12 @@ static void session_thread_set_tls_interaction_cb (ThreadClosure *thread_closure, gpointer data) { - GTlsInteraction *interaction = data; + GTlsCertificate *cert = data; + glnx_unref_object OstreeTlsCertInteraction *interaction = NULL; + + /* The GTlsInteraction instance must be created in the + * session thread so it uses the correct GMainContext. */ + interaction = _ostree_tls_cert_interaction_new (cert); g_object_set (thread_closure->session, SOUP_SESSION_TLS_INTERACTION, @@ -645,7 +650,7 @@ _ostree_fetcher_set_client_cert (OstreeFetcher *self, #ifdef HAVE_LIBSOUP_CLIENT_CERTS session_thread_idle_add (self->thread_closure, session_thread_set_tls_interaction_cb, - _ostree_tls_cert_interaction_new (cert), + g_object_ref (cert), (GDestroyNotify) g_object_unref); #else g_warning ("This version of OSTree is compiled without client side certificate support"); diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index ad309e75..0879f44b 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -32,6 +32,8 @@ #include "ostree-core-private.h" #include "ostree-repo-private.h" +#define WHITEOUT_PREFIX ".wh." + static gboolean checkout_object_for_uncompressed_cache (OstreeRepo *self, const char *loose_path, @@ -102,9 +104,16 @@ checkout_object_for_uncompressed_cache (OstreeRepo *self, return ret; } +static gboolean +fsync_is_enabled (OstreeRepo *self, + OstreeRepoCheckoutOptions *options) +{ + return !(self->disable_fsync || options->disable_fsync); +} + static gboolean write_regular_file_content (OstreeRepo *self, - OstreeRepoCheckoutMode mode, + OstreeRepoCheckoutOptions *options, GOutputStream *output, GFileInfo *file_info, GVariant *xattrs, @@ -113,6 +122,7 @@ write_regular_file_content (OstreeRepo *self, GError **error) { gboolean ret = FALSE; + const OstreeRepoCheckoutMode mode = options->mode; int fd; int res; @@ -154,12 +164,12 @@ write_regular_file_content (OstreeRepo *self, } } - if (!self->disable_fsync) + if (fsync_is_enabled (self, options)) { if (fsync (fd) == -1) { gs_set_error_from_errno (error, errno); - goto out; + goto out; } } @@ -238,7 +248,7 @@ checkout_file_from_input_at (OstreeRepo *self, temp_out = g_unix_output_stream_new (fd, TRUE); fd = -1; /* Transfer ownership */ - if (!write_regular_file_content (self, options->mode, temp_out, file_info, xattrs, input, + if (!write_regular_file_content (self, options, temp_out, file_info, xattrs, input, cancellable, error)) goto out; } @@ -298,7 +308,7 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo, cancellable, error)) goto out; - if (!write_regular_file_content (repo, options->mode, temp_out, file_info, xattrs, input, + if (!write_regular_file_content (repo, options, temp_out, file_info, xattrs, input, cancellable, error)) goto out; } @@ -388,20 +398,46 @@ checkout_one_file_at (OstreeRepo *repo, const char *checksum; gboolean is_symlink; gboolean can_cache; - gboolean did_hardlink = FALSE; + gboolean need_copy = TRUE; char loose_path_buf[_OSTREE_LOOSE_PATH_MAX]; g_autoptr(GInputStream) input = NULL; g_autoptr(GVariant) xattrs = NULL; + gboolean is_whiteout; is_symlink = g_file_info_get_file_type (source_info) == G_FILE_TYPE_SYMBOLIC_LINK; checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)source); - /* Try to do a hardlink first, if it's a regular file. This also - * traverses all parent repos. + is_whiteout = !is_symlink && options->process_whiteouts && + g_str_has_prefix (destination_name, WHITEOUT_PREFIX); + + /* First, see if it's a Docker whiteout, + * https://github.com/docker/docker/blob/1a714e76a2cb9008cd19609059e9988ff1660b78/pkg/archive/whiteouts.go */ - if (!is_symlink) + if (is_whiteout) { + const char *name = destination_name + (sizeof (WHITEOUT_PREFIX) - 1); + + if (!name[0]) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid empty whiteout '%s'", name); + goto out; + } + + g_assert (name[0] != '/'); /* Sanity */ + + if (!glnx_shutil_rm_rf_at (destination_dfd, name, cancellable, error)) + goto out; + + need_copy = FALSE; + } + else if (!is_symlink) + { + gboolean did_hardlink = FALSE; + /* Try to do a hardlink first, if it's a regular file. This also + * traverses all parent repos. + */ OstreeRepo *current_repo = repo; while (current_repo) @@ -454,6 +490,8 @@ checkout_one_file_at (OstreeRepo *repo, } current_repo = current_repo->parent_repo; } + + need_copy = !did_hardlink; } can_cache = (options->enable_uncompressed_cache @@ -463,11 +501,14 @@ checkout_one_file_at (OstreeRepo *repo, * it now, stick it in the cache, and then hardlink to that. */ if (can_cache + && !is_whiteout && !is_symlink - && !did_hardlink + && need_copy && repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && options->mode == OSTREE_REPO_CHECKOUT_MODE_USER) { + gboolean did_hardlink; + if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL, cancellable, error)) goto out; @@ -518,10 +559,12 @@ checkout_one_file_at (OstreeRepo *repo, g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name); goto out; } + + need_copy = !did_hardlink; } /* Fall back to copy if we couldn't hardlink */ - if (!did_hardlink) + if (need_copy) { if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs, cancellable, error)) @@ -726,17 +769,13 @@ checkout_tree_at (OstreeRepo *self, } } - /* Finally, fsync to ensure all entries are on disk. Ultimately - * this should be configurable for the case where we're constructing - * buildroots. - */ - if (!self->disable_fsync) + if (fsync_is_enabled (self, options)) { - if (fsync (destination_dfd) == -1) - { - gs_set_error_from_errno (error, errno); - goto out; - } + if (fsync (destination_dfd) == -1) + { + gs_set_error_from_errno (error, errno); + goto out; + } } ret = TRUE; diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 09190285..a7c16192 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -412,12 +412,12 @@ add_size_index_to_metadata (OstreeRepo *self, for (i = 0; i < sorted_keys->len; i++) { - guint8 csum[32]; + guint8 csum[OSTREE_SHA256_DIGEST_LEN]; const char *e_checksum = sorted_keys->pdata[i]; GString *buffer = g_string_new (NULL); ostree_checksum_inplace_to_bytes (e_checksum, csum); - g_string_append_len (buffer, (char*)csum, 32); + g_string_append_len (buffer, (char*)csum, sizeof (csum)); e_size = g_hash_table_lookup (self->object_sizes, e_checksum); _ostree_write_varuint64 (buffer, e_size->archived); @@ -1144,7 +1144,8 @@ ostree_repo_scan_hardlinks (OstreeRepo *self, * ostree_repo_prepare_transaction: * @self: An #OstreeRepo * @out_transaction_resume: (allow-none) (out): Whether this transaction - * is resuming from a previous one. + * is resuming from a previous one. This is a legacy state, now OSTree + * pulls use per-commit `state/.commitpartial` files. * @cancellable: Cancellable * @error: Error * diff --git a/src/libostree/ostree-repo-libarchive.c b/src/libostree/ostree-repo-libarchive.c index e86e4026..1c051592 100644 --- a/src/libostree/ostree-repo-libarchive.c +++ b/src/libostree/ostree-repo-libarchive.c @@ -24,6 +24,7 @@ #include "ostree-core-private.h" #include "ostree-repo-private.h" +#include "ostree-repo-file.h" #include "ostree-mutable-tree.h" #ifdef HAVE_LIBARCHIVE @@ -77,6 +78,7 @@ file_info_from_archive_entry_and_modifier (OstreeRepo *repo, static gboolean import_libarchive_entry_file (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, struct archive *a, struct archive_entry *entry, GFileInfo *file_info, @@ -92,8 +94,27 @@ import_libarchive_entry_file (OstreeRepo *self, if (g_cancellable_set_error_if_cancelled (cancellable, error)) return FALSE; - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) - archive_stream = _ostree_libarchive_input_stream_new (a); + switch (g_file_info_get_file_type (file_info)) + { + case G_FILE_TYPE_REGULAR: + archive_stream = _ostree_libarchive_input_stream_new (a); + break; + case G_FILE_TYPE_SYMBOLIC_LINK: + break; + default: + if (opts->ignore_unsupported_content) + { + ret = TRUE; + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Unable to import non-regular/non-symlink file '%s'", + archive_entry_pathname (entry)); + goto out; + } + } if (!ostree_raw_file_to_content_stream (archive_stream, file_info, NULL, &file_object_input, &length, cancellable, error)) @@ -110,6 +131,7 @@ import_libarchive_entry_file (OstreeRepo *self, static gboolean write_libarchive_entry_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, OstreeMutableTree *root, struct archive *a, struct archive_entry *entry, @@ -248,16 +270,19 @@ write_libarchive_entry_to_mtree (OstreeRepo *self, goto out; } - if (!import_libarchive_entry_file (self, a, entry, file_info, &tmp_csum, + if (!import_libarchive_entry_file (self, opts, a, entry, file_info, &tmp_csum, cancellable, error)) goto out; - - g_free (tmp_checksum); - tmp_checksum = ostree_checksum_from_bytes (tmp_csum); - if (!ostree_mutable_tree_replace_file (parent, basename, - tmp_checksum, - error)) - goto out; + + if (tmp_csum) + { + g_free (tmp_checksum); + tmp_checksum = ostree_checksum_from_bytes (tmp_csum); + if (!ostree_mutable_tree_replace_file (parent, basename, + tmp_checksum, + error)) + goto out; + } } } @@ -266,6 +291,109 @@ write_libarchive_entry_to_mtree (OstreeRepo *self, return ret; } #endif + +static gboolean +create_empty_dir_with_uidgid (OstreeRepo *self, + guint32 uid, + guint32 gid, + guint8 **out_csum, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GFileInfo) tmp_dir_info = g_file_info_new (); + + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", uid); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", gid); + g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); + + return _ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, out_csum, cancellable, error); +} + +/** + * ostree_repo_import_archive_to_mtree: + * @self: An #OstreeRepo + * @opts: Options structure, ensure this is zeroed, then set specific variables + * @archive: Really this is "struct archive*" + * @mtree: The #OstreeMutableTree to write to + * @modifier: (allow-none): Optional commit modifier + * @cancellable: Cancellable + * @error: Error + * + * Import an archive file @archive into the repository, and write its + * file structure to @mtree. + */ +gboolean +ostree_repo_import_archive_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, + void *archive, + OstreeMutableTree *mtree, + OstreeRepoCommitModifier *modifier, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + struct archive *a = archive; + struct archive_entry *entry; + g_autofree guchar *tmp_csum = NULL; + int r; + + + while (TRUE) + { + r = archive_read_next_header (a, &entry); + if (r == ARCHIVE_EOF) + break; + else if (r != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + /* TODO - refactor this to only create the metadata on demand + * (i.e. if there is a missing parent dir) + */ + if (opts->autocreate_parents && !tmp_csum) + { + /* Here, we auto-pick the first uid/gid we find in the + * archive. Realistically this is probably always going to + * be root, but eh, at least we try to match. + */ + if (!create_empty_dir_with_uidgid (self, archive_entry_uid (entry), + archive_entry_gid (entry), + &tmp_csum, cancellable, error)) + goto out; + } + + if (!write_libarchive_entry_to_mtree (self, opts, mtree, a, + entry, modifier, tmp_csum, + cancellable, error)) + goto out; + } + + /* If we didn't import anything at all, and autocreation of parents + * is enabled, automatically create a root directory. This is + * useful primarily when importing Docker image layers, which can + * just be metadata. + */ + if (!ostree_mutable_tree_get_metadata_checksum (mtree) && opts->autocreate_parents) + { + char tmp_checksum[65]; + + if (!tmp_csum) + { + /* We didn't have any archive entries to match, so pick uid 0, gid 0. */ + if (!create_empty_dir_with_uidgid (self, 0, 0, &tmp_csum, cancellable, error)) + goto out; + } + + ostree_checksum_inplace_from_bytes (tmp_csum, tmp_checksum); + ostree_mutable_tree_set_metadata_checksum (mtree, tmp_checksum); + } + + ret = TRUE; + out: + return ret; +} /** * ostree_repo_write_archive_to_mtree: @@ -292,10 +420,8 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, #ifdef HAVE_LIBARCHIVE gboolean ret = FALSE; struct archive *a = NULL; - struct archive_entry *entry; - int r; g_autoptr(GFileInfo) tmp_dir_info = NULL; - g_autofree guchar *tmp_csum = NULL; + OstreeRepoImportArchiveOptions opts = { 0, }; a = archive_read_new (); #ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL @@ -310,35 +436,11 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, goto out; } - while (TRUE) - { - r = archive_read_next_header (a, &entry); - if (r == ARCHIVE_EOF) - break; - else if (r != ARCHIVE_OK) - { - propagate_libarchive_error (error, a); - goto out; - } + opts.autocreate_parents = !!autocreate_parents; - if (autocreate_parents && !tmp_csum) - { - tmp_dir_info = g_file_info_new (); - - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::uid", archive_entry_uid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::gid", archive_entry_gid (entry)); - g_file_info_set_attribute_uint32 (tmp_dir_info, "unix::mode", 0755 | S_IFDIR); - - if (!_ostree_repo_write_directory_meta (self, tmp_dir_info, NULL, &tmp_csum, cancellable, error)) - goto out; - } + if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error)) + goto out; - if (!write_libarchive_entry_to_mtree (self, mtree, a, - entry, modifier, - autocreate_parents ? tmp_csum : NULL, - cancellable, error)) - goto out; - } if (archive_read_close (a) != ARCHIVE_OK) { propagate_libarchive_error (error, a); @@ -356,3 +458,251 @@ ostree_repo_write_archive_to_mtree (OstreeRepo *self, return FALSE; #endif } + +#ifdef HAVE_LIBARCHIVE + +static gboolean +file_to_archive_entry_common (GFile *root, + OstreeRepoExportArchiveOptions *opts, + GFile *path, + GFileInfo *file_info, + struct archive_entry *entry, + GError **error) +{ + gboolean ret = FALSE; + g_autofree char *pathstr = g_file_get_relative_path (root, path); + g_autoptr(GVariant) xattrs = NULL; + time_t ts = (time_t) opts->timestamp_secs; + + if (pathstr && !pathstr[0]) + { + g_free (pathstr); + pathstr = g_strdup ("."); + } + + archive_entry_update_pathname_utf8 (entry, pathstr); + archive_entry_set_ctime (entry, ts, 0); + archive_entry_set_mtime (entry, ts, 0); + archive_entry_set_atime (entry, ts, 0); + archive_entry_set_uid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::uid")); + archive_entry_set_gid (entry, g_file_info_get_attribute_uint32 (file_info, "unix::gid")); + archive_entry_set_mode (entry, g_file_info_get_attribute_uint32 (file_info, "unix::mode")); + + if (!ostree_repo_file_get_xattrs ((OstreeRepoFile*)path, &xattrs, NULL, error)) + goto out; + + if (!opts->disable_xattrs) + { + int i, n; + + n = g_variant_n_children (xattrs); + for (i = 0; i < n; i++) + { + const guint8* name; + g_autoptr(GVariant) value = NULL; + const guint8* value_data; + gsize value_len; + + g_variant_get_child (xattrs, i, "(^&ay@ay)", &name, &value); + value_data = g_variant_get_fixed_array (value, &value_len, 1); + + archive_entry_xattr_add_entry (entry, (char*)name, + (char*) value_data, value_len); + } + } + + ret = TRUE; + out: + return ret; +} + +static gboolean +write_header_free_entry (struct archive *a, + struct archive_entry **entryp, + GError **error) +{ + struct archive_entry *entry = *entryp; + gboolean ret = FALSE; + + if (archive_write_header (a, entry) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + ret = TRUE; + out: + archive_entry_free (entry); + *entryp = NULL; + return ret; +} + +static gboolean +write_directory_to_libarchive_recurse (OstreeRepo *self, + OstreeRepoExportArchiveOptions *opts, + GFile *root, + GFile *dir, + struct archive *a, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + g_autoptr(GFileInfo) dir_info = NULL; + g_autoptr(GFileEnumerator) dir_enum = NULL; + struct archive_entry *entry = NULL; + + dir_info = g_file_query_info (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!dir_info) + goto out; + + entry = archive_entry_new2 (a); + if (!file_to_archive_entry_common (root, opts, dir, dir_info, entry, error)) + goto out; + if (!write_header_free_entry (a, &entry, error)) + goto out; + + dir_enum = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, + G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, + cancellable, error); + if (!dir_enum) + goto out; + + while (TRUE) + { + GFileInfo *file_info; + GFile *path; + + if (!gs_file_enumerator_iterate (dir_enum, &file_info, &path, + cancellable, error)) + goto out; + if (file_info == NULL) + break; + + /* First, handle directories recursively */ + if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) + { + if (!write_directory_to_libarchive_recurse (self, opts, root, path, a, + cancellable, error)) + goto out; + + /* Go to the next entry */ + continue; + } + + /* Past here, should be a regular file or a symlink */ + + entry = archive_entry_new2 (a); + if (!file_to_archive_entry_common (root, opts, path, file_info, entry, error)) + goto out; + + switch (g_file_info_get_file_type (file_info)) + { + case G_FILE_TYPE_SYMBOLIC_LINK: + { + archive_entry_set_symlink (entry, g_file_info_get_symlink_target (file_info)); + if (!write_header_free_entry (a, &entry, error)) + goto out; + } + break; + case G_FILE_TYPE_REGULAR: + { + guint8 buf[8192]; + g_autoptr(GInputStream) file_in = NULL; + g_autoptr(GFileInfo) file_info = NULL; + const char *checksum; + + checksum = ostree_repo_file_get_checksum ((OstreeRepoFile*)path); + + if (!ostree_repo_load_file (self, checksum, &file_in, &file_info, NULL, + cancellable, error)) + goto out; + + archive_entry_set_size (entry, g_file_info_get_size (file_info)); + + if (archive_write_header (a, entry) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + while (TRUE) + { + gssize bytes_read = g_input_stream_read (file_in, buf, sizeof (buf), + cancellable, error); + if (bytes_read < 0) + goto out; + if (bytes_read == 0) + break; + + { ssize_t r = archive_write_data (a, buf, bytes_read); + if (r != bytes_read) + { + propagate_libarchive_error (error, a); + g_prefix_error (error, "Failed to write %" G_GUINT64_FORMAT " bytes (code %" G_GUINT64_FORMAT"): ", (guint64)bytes_read, (guint64)r); + goto out; + } + } + } + + if (archive_write_finish_entry (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + + archive_entry_free (entry); + entry = NULL; + } + break; + default: + g_assert_not_reached (); + } + } + + ret = TRUE; + out: + if (entry) + archive_entry_free (entry); + return ret; +} +#endif + +/** + * ostree_repo_export_tree_to_archive: + * @self: An #OstreeRepo + * @opts: Options controlling conversion + * @root: An #OstreeRepoFile for the base directory + * @archive: A `struct archive`, but specified as void to avoid a dependency on the libarchive headers + * @cancellable: Cancellable + * @error: Error + * + * Import an archive file @archive into the repository, and write its + * file structure to @mtree. + */ +gboolean +ostree_repo_export_tree_to_archive (OstreeRepo *self, + OstreeRepoExportArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, + GCancellable *cancellable, + GError **error) +{ +#ifdef HAVE_LIBARCHIVE + gboolean ret = FALSE; + struct archive *a = archive; + + if (!write_directory_to_libarchive_recurse (self, opts, (GFile*)root, (GFile*)root, + a, cancellable, error)) + goto out; + + ret = TRUE; + out: + return ret; +#else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree is not compiled with libarchive support"); + return FALSE; +#endif +} diff --git a/src/libostree/ostree-repo-private.h b/src/libostree/ostree-repo-private.h index b6ea3177..463b3dd7 100644 --- a/src/libostree/ostree-repo-private.h +++ b/src/libostree/ostree-repo-private.h @@ -50,8 +50,6 @@ struct OstreeRepo { int repo_dir_fd; GFile *tmp_dir; int tmp_dir_fd; - GFile *local_heads_dir; - GFile *remote_heads_dir; GFile *objects_dir; GFile *state_dir; int objects_dir_fd; diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 907ed454..0be1b380 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -48,7 +48,9 @@ typedef struct { GCancellable *cancellable; OstreeAsyncProgress *progress; - gboolean transaction_resuming; + gboolean dry_run; + gboolean dry_run_emitted_progress; + gboolean legacy_transaction_resuming; enum { OSTREE_PULL_PHASE_FETCHING_REFS, OSTREE_PULL_PHASE_FETCHING_OBJECTS @@ -78,6 +80,7 @@ typedef struct { guint n_outstanding_deltapart_write_requests; guint n_total_deltaparts; guint64 total_deltapart_size; + guint64 total_deltapart_usize; gint n_requested_metadata; gint n_requested_content; guint n_fetched_deltaparts; @@ -123,7 +126,7 @@ typedef struct { } FetchStaticDeltaData; typedef struct { - guchar csum[32]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; OstreeObjectType objtype; guint recursion_depth; } ScanObjectQueueData; @@ -227,6 +230,8 @@ update_progress (gpointer user_data) pull_data->n_total_deltaparts); ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size", pull_data->total_deltapart_size); + ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize", + pull_data->total_deltapart_usize); ostree_async_progress_set_uint (pull_data->progress, "total-delta-superblocks", pull_data->static_delta_superblocks->len); @@ -243,6 +248,9 @@ update_progress (gpointer user_data) else ostree_async_progress_set_status (pull_data->progress, NULL); + if (pull_data->dry_run) + pull_data->dry_run_emitted_progress = TRUE; + return TRUE; } @@ -262,6 +270,9 @@ pull_termination_condition (OtPullData *pull_data) if (pull_data->caught_error) return TRUE; + if (pull_data->dry_run) + return pull_data->dry_run_emitted_progress; + switch (pull_data->phase) { case OSTREE_PULL_PHASE_FETCHING_REFS: @@ -935,8 +946,7 @@ static_deltapart_fetch_on_complete (GObject *object, g_autoptr(GVariant) metadata = NULL; g_autofree char *temp_path = NULL; g_autoptr(GInputStream) in = NULL; - g_autofree char *actual_checksum = NULL; - g_autofree guint8 *csum = NULL; + g_autoptr(GVariant) part = NULL; GError *local_error = NULL; GError **error = &local_error; gs_fd_close int fd = -1; @@ -950,54 +960,33 @@ static_deltapart_fetch_on_complete (GObject *object, fd = openat (_ostree_fetcher_get_dfd (fetcher), temp_path, O_RDONLY | O_CLOEXEC); if (fd == -1) { - gs_set_error_from_errno (error, errno); + glnx_set_error_from_errno (error); goto out; } + + /* From here on, if we fail to apply the delta, we'll re-fetch it */ + if (unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0) < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + in = g_unix_input_stream_new (fd, FALSE); - /* TODO - consider making async */ - if (!ot_gio_checksum_stream (in, &csum, pull_data->cancellable, error)) + /* TODO - make async */ + if (!_ostree_static_delta_part_open (in, NULL, 0, fetch_data->expected_checksum, + &part, pull_data->cancellable, error)) goto out; - actual_checksum = ostree_checksum_from_bytes (csum); - - if (strcmp (actual_checksum, fetch_data->expected_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted static delta part; checksum expected='%s' actual='%s'", - fetch_data->expected_checksum, actual_checksum); - goto out; - } - - /* Might as well close the fd here */ - (void) g_input_stream_close (in, NULL, NULL); - - { - GMappedFile *mfile = NULL; - g_autoptr(GBytes) delta_data = NULL; - - mfile = g_mapped_file_new_from_fd (fd, FALSE, error); - if (!mfile) - goto out; - delta_data = g_mapped_file_get_bytes (mfile); - g_mapped_file_unref (mfile); - - /* Unlink now while we're holding an open fd, so that on success - * or error, the file will be gone. This is particularly - * important if say we hit e.g. ENOSPC. - */ - (void) unlinkat (_ostree_fetcher_get_dfd (fetcher), temp_path, 0); - - _ostree_static_delta_part_execute_async (pull_data->repo, - fetch_data->objects, - delta_data, - /* Trust checksums if summary was gpg signed */ - pull_data->gpg_verify_summary && pull_data->summary_data_sig, - pull_data->cancellable, - on_static_delta_written, - fetch_data); - pull_data->n_outstanding_deltapart_write_requests++; - } + _ostree_static_delta_part_execute_async (pull_data->repo, + fetch_data->objects, + part, + /* Trust checksums if summary was gpg signed */ + pull_data->gpg_verify_summary && pull_data->summary_data_sig, + pull_data->cancellable, + on_static_delta_written, + fetch_data); + pull_data->n_outstanding_deltapart_write_requests++; out: g_assert (pull_data->n_outstanding_deltapart_fetches > 0); @@ -1150,7 +1139,7 @@ queue_scan_one_metadata_object (OtPullData *pull_data, OstreeObjectType objtype, guint recursion_depth) { - guchar buf[32]; + guchar buf[OSTREE_SHA256_DIGEST_LEN]; ostree_checksum_inplace_to_bytes (csum, buf); queue_scan_one_metadata_object_c (pull_data, buf, objtype, recursion_depth); } @@ -1227,7 +1216,7 @@ scan_one_metadata_object_c (OtPullData *pull_data, } else if (is_stored) { - gboolean do_scan = pull_data->transaction_resuming || is_requested || pull_data->commitpartial_exists; + gboolean do_scan = pull_data->legacy_transaction_resuming || is_requested || pull_data->commitpartial_exists; /* For commits, always refetch detached metadata. */ if (objtype == OSTREE_OBJECT_TYPE_COMMIT) @@ -1454,6 +1443,7 @@ request_static_delta_superblock_sync (OtPullData *pull_data, static gboolean process_one_static_delta_fallback (OtPullData *pull_data, + gboolean delta_byteswap, GVariant *fallback_object, GCancellable *cancellable, GError **error) @@ -1473,10 +1463,20 @@ process_one_static_delta_fallback (OtPullData *pull_data, if (!ostree_validate_structureof_csum_v (csum_v, error)) goto out; - objtype = (OstreeObjectType)objtype_y; - checksum = ostree_checksum_from_bytes_v (csum_v); + compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size); + uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size); pull_data->total_deltapart_size += compressed_size; + pull_data->total_deltapart_usize += uncompressed_size; + + if (pull_data->dry_run) + { + ret = TRUE; + goto out; + } + + objtype = (OstreeObjectType)objtype_y; + checksum = ostree_checksum_from_bytes_v (csum_v); if (!ostree_repo_has_object (pull_data->repo, objtype, checksum, &is_stored, @@ -1522,11 +1522,14 @@ process_one_static_delta (OtPullData *pull_data, GError **error) { gboolean ret = FALSE; + gboolean delta_byteswap; g_autoptr(GVariant) metadata = NULL; g_autoptr(GVariant) headers = NULL; g_autoptr(GVariant) fallback_objects = NULL; guint i, n; + delta_byteswap = _ostree_delta_needs_byteswap (delta_superblock); + /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ metadata = g_variant_get_child_value (delta_superblock, 0); headers = g_variant_get_child_value (delta_superblock, 6); @@ -1539,13 +1542,14 @@ process_one_static_delta (OtPullData *pull_data, g_autoptr(GVariant) fallback_object = g_variant_get_child_value (fallback_objects, i); - if (!process_one_static_delta_fallback (pull_data, + if (!process_one_static_delta_fallback (pull_data, delta_byteswap, fallback_object, cancellable, error)) goto out; } /* Write the to-commit object */ + if (!pull_data->dry_run) { g_autoptr(GVariant) to_csum_v = NULL; g_autofree char *to_checksum = NULL; @@ -1604,14 +1608,18 @@ process_one_static_delta (OtPullData *pull_data, FetchStaticDeltaData *fetch_data; g_autoptr(GVariant) csum_v = NULL; g_autoptr(GVariant) objects = NULL; - g_autoptr(GVariant) part_data = NULL; - g_autoptr(GBytes) delta_data = NULL; + g_autoptr(GBytes) inline_part_bytes = NULL; guint64 size, usize; guint32 version; + const gboolean trusted = pull_data->gpg_verify_summary && pull_data->summary_data_sig; header = g_variant_get_child_value (headers, i); g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); + version = maybe_swap_endian_u32 (delta_byteswap, version); + size = maybe_swap_endian_u64 (delta_byteswap, size); + usize = maybe_swap_endian_u64 (delta_byteswap, usize); + if (version > OSTREE_DELTAPART_VERSION) { g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, @@ -1623,31 +1631,6 @@ process_one_static_delta (OtPullData *pull_data, if (!csum) goto out; - deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i); - - part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)")); - if (part_data) - { - g_autofree char *actual_checksum = NULL; - g_autofree char *expected_checksum = ostree_checksum_from_bytes_v (csum_v); - - delta_data = g_variant_get_data_as_bytes (part_data); - - /* For inline parts we are relying on per-commit GPG, so this isn't strictly necessary for security. - * See https://github.com/GNOME/ostree/pull/139 - */ - actual_checksum = g_compute_checksum_for_bytes (G_CHECKSUM_SHA256, delta_data); - if (strcmp (actual_checksum, expected_checksum) != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted static delta part; checksum expected='%s' actual='%s'", - expected_checksum, actual_checksum); - goto out; - } - } - - pull_data->total_deltapart_size += size; - if (!_ostree_repo_static_delta_part_have_all_objects (pull_data->repo, objects, &have_all, @@ -1663,18 +1646,42 @@ process_one_static_delta (OtPullData *pull_data, continue; } + deltapart_path = _ostree_get_relative_static_delta_part_path (from_revision, to_revision, i); + + { g_autoptr(GVariant) part_datav = + g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE ("(yay)")); + + if (part_datav) + inline_part_bytes = g_variant_get_data_as_bytes (part_datav); + } + + pull_data->total_deltapart_size += size; + pull_data->total_deltapart_usize += usize; + + if (pull_data->dry_run) + continue; + fetch_data = g_new0 (FetchStaticDeltaData, 1); fetch_data->pull_data = pull_data; fetch_data->objects = g_variant_ref (objects); fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v); - if (delta_data != NULL) + if (inline_part_bytes != NULL) { + g_autoptr(GInputStream) memin = g_memory_input_stream_new_from_bytes (inline_part_bytes); + g_autoptr(GVariant) inline_delta_part = NULL; + + /* For inline parts we are relying on per-commit GPG, so don't bother checksumming. */ + if (!_ostree_static_delta_part_open (memin, inline_part_bytes, + OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM, + NULL, &inline_delta_part, + cancellable, error)) + goto out; + _ostree_static_delta_part_execute_async (pull_data->repo, fetch_data->objects, - delta_data, - /* Trust checksums if summary was gpg signed */ - pull_data->gpg_verify_summary && pull_data->summary_data_sig, + inline_delta_part, + trusted, pull_data->cancellable, on_static_delta_written, fetch_data); @@ -1789,8 +1796,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, OstreeRepoPullFlags flags = 0; const char *dir_to_pull = NULL; char **refs_to_fetch = NULL; + char **override_commit_ids = NULL; GSource *update_timeout = NULL; gboolean disable_static_deltas = FALSE; + gboolean require_static_deltas = FALSE; if (options) { @@ -1803,13 +1812,24 @@ ostree_repo_pull_with_options (OstreeRepo *self, (void) g_variant_lookup (options, "override-remote-name", "s", &pull_data->remote_name); (void) g_variant_lookup (options, "depth", "i", &pull_data->maxdepth); (void) g_variant_lookup (options, "disable-static-deltas", "b", &disable_static_deltas); + (void) g_variant_lookup (options, "require-static-deltas", "b", &require_static_deltas); + (void) g_variant_lookup (options, "override-commit-ids", "^a&s", &override_commit_ids); + (void) g_variant_lookup (options, "dry-run", "b", &pull_data->dry_run); } g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE); + if (refs_to_fetch && override_commit_ids) + g_return_val_if_fail (g_strv_length (refs_to_fetch) == g_strv_length (override_commit_ids), FALSE); if (dir_to_pull) g_return_val_if_fail (dir_to_pull[0] == '/', FALSE); + g_return_val_if_fail (!(disable_static_deltas && require_static_deltas), FALSE); + /* We only do dry runs with static deltas, because we don't really have any + * in-advance information for bare fetches. + */ + g_return_val_if_fail (!pull_data->dry_run || require_static_deltas, FALSE); + pull_data->is_mirror = (flags & OSTREE_REPO_PULL_FLAGS_MIRROR) > 0; pull_data->is_commit_only = (flags & OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY) > 0; @@ -1992,6 +2012,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, goto out; } + if (!bytes_summary && require_static_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Fetch configured to require static deltas, but no summary found"); + goto out; + } + if (bytes_summary) { uri = suburi_new (pull_data->base_uri, "summary.sig", NULL); @@ -2067,7 +2094,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, { const char *delta; GVariant *csum_v = NULL; - guchar *csum_data = g_malloc (32); + guchar *csum_data = g_malloc (OSTREE_SHA256_DIGEST_LEN); g_autoptr(GVariant) ref = g_variant_get_child_value (deltas, i); g_variant_get_child (ref, 0, "&s", &delta); @@ -2096,8 +2123,10 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else if (refs_to_fetch != NULL) { - char **strviter; - for (strviter = refs_to_fetch; *strviter; strviter++) + char **strviter = refs_to_fetch; + char **commitid_strviter = override_commit_ids ? override_commit_ids : NULL; + + while (*strviter) { const char *branch = *strviter; @@ -2108,8 +2137,13 @@ ostree_repo_pull_with_options (OstreeRepo *self, } else { - g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL); + char *commitid = commitid_strviter ? g_strdup (*commitid_strviter) : NULL; + g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), commitid); } + + strviter++; + if (commitid_strviter) + commitid_strviter++; } } else @@ -2136,28 +2170,36 @@ ostree_repo_pull_with_options (OstreeRepo *self, while (g_hash_table_iter_next (&hash_iter, &key, &value)) { const char *branch = key; + const char *override_commitid = value; char *contents = NULL; - if (pull_data->summary) + /* Support specifying "" for an override commitid */ + if (override_commitid && *override_commitid) { - gsize commit_size = 0; - guint64 *malloced_size; - - if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) - goto out; - - malloced_size = g_new0 (guint64, 1); - *malloced_size = commit_size; - g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size); + g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), g_strdup (override_commitid)); } - else + else { - if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) - goto out; + if (pull_data->summary) + { + gsize commit_size = 0; + guint64 *malloced_size; + + if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) + goto out; + + malloced_size = g_new0 (guint64, 1); + *malloced_size = commit_size; + g_hash_table_insert (pull_data->expected_commit_sizes, g_strdup (contents), malloced_size); + } + else + { + if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) + goto out; + } + /* Transfer ownership of contents */ + g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); } - - /* Transfer ownership of contents */ - g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); } /* Create the state directory here - it's new with the commitpartial code, @@ -2182,11 +2224,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->fetcher == NULL) goto out; - if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->transaction_resuming, + if (!ostree_repo_prepare_transaction (pull_data->repo, &pull_data->legacy_transaction_resuming, cancellable, error)) goto out; - g_debug ("resuming transaction: %s", pull_data->transaction_resuming ? "true" : " false"); + if (pull_data->legacy_transaction_resuming) + g_debug ("resuming legacy transaction"); g_hash_table_iter_init (&hash_iter, commits_to_fetch); while (g_hash_table_iter_next (&hash_iter, &key, &value)) @@ -2207,17 +2250,22 @@ ostree_repo_pull_with_options (OstreeRepo *self, &from_revision, error)) goto out; -#ifdef BUILDOPT_STATIC_DELTAS if (!disable_static_deltas && (from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0)) { if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision, &delta_superblock, cancellable, error)) goto out; } -#endif if (!delta_superblock) { + if (require_static_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Static deltas required, but none found for %s to %s", + from_revision, to_revision); + goto out; + } g_debug ("no delta superblock for %s-%s", from_revision ? from_revision : "empty", to_revision); queue_scan_one_metadata_object (pull_data, to_revision, OSTREE_OBJECT_TYPE_COMMIT, 0); } @@ -2234,7 +2282,7 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->progress) { - update_timeout = g_timeout_source_new_seconds (1); + update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1); g_source_set_priority (update_timeout, G_PRIORITY_HIGH); g_source_set_callback (update_timeout, update_progress, pull_data, NULL); g_source_attach (update_timeout, pull_data->main_context); @@ -2247,6 +2295,12 @@ ostree_repo_pull_with_options (OstreeRepo *self, if (pull_data->caught_error) goto out; + + if (pull_data->dry_run) + { + ret = TRUE; + goto out; + } g_assert_cmpint (pull_data->n_outstanding_metadata_fetches, ==, 0); g_assert_cmpint (pull_data->n_outstanding_metadata_write_requests, ==, 0); diff --git a/src/libostree/ostree-repo-refs.c b/src/libostree/ostree-repo-refs.c index 6d3bd51e..68e34f20 100644 --- a/src/libostree/ostree-repo-refs.c +++ b/src/libostree/ostree-repo-refs.c @@ -25,19 +25,19 @@ static gboolean add_ref_to_set (const char *remote, - GFile *base, - GFile *child, + int base_fd, + const char *path, GHashTable *refs, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; char *contents; - char *relpath; gsize len; GString *refname; - if (!g_file_load_contents (child, cancellable, &contents, &len, NULL, error)) + contents = glnx_file_get_contents_utf8_at (base_fd, path, &len, cancellable, error); + if (!contents) goto out; g_strchomp (contents); @@ -48,9 +48,7 @@ add_ref_to_set (const char *remote, g_string_append (refname, remote); g_string_append_c (refname, ':'); } - relpath = g_file_get_relative_path (base, child); - g_string_append (refname, relpath); - g_free (relpath); + g_string_append (refname, path); g_hash_table_insert (refs, g_string_free (refname, FALSE), contents); @@ -116,44 +114,69 @@ write_checksum_file_at (OstreeRepo *self, return ret; } +static gboolean +openat_ignore_enoent (int dfd, + const char *path, + int *out_fd, + GError **error) +{ + gboolean ret = FALSE; + int target_fd = -1; + + target_fd = openat (dfd, path, O_CLOEXEC | O_RDONLY); + if (target_fd < 0) + { + if (errno != ENOENT) + { + glnx_set_error_from_errno (error); + goto out; + } + } + + ret = TRUE; + *out_fd = target_fd; + out: + return ret; +} + static gboolean find_ref_in_remotes (OstreeRepo *self, const char *rev, - GFile **out_file, + int *out_fd, GError **error) { gboolean ret = FALSE; - g_autoptr(GFileEnumerator) dir_enum = NULL; - g_autoptr(GFile) ret_file = NULL; + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + glnx_fd_close int ret_fd = -1; - dir_enum = g_file_enumerate_children (self->remote_heads_dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - NULL, error); - if (!dir_enum) + if (!glnx_dirfd_iterator_init_at (self->repo_dir_fd, "refs/remotes", TRUE, &dfd_iter, error)) goto out; while (TRUE) { - GFileInfo *file_info; - GFile *child; - if (!gs_file_enumerator_iterate (dir_enum, &file_info, &child, - NULL, error)) + struct dirent *dent = NULL; + glnx_fd_close int remote_dfd = -1; + + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, NULL, error)) goto out; - if (file_info == NULL) + if (dent == NULL) break; - if (g_file_info_get_file_type (file_info) != G_FILE_TYPE_DIRECTORY) + + if (dent->d_type != DT_DIR) continue; - g_clear_object (&ret_file); - ret_file = g_file_resolve_relative_path (child, rev); - if (!g_file_query_exists (ret_file, NULL)) - g_clear_object (&ret_file); - else + if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &remote_dfd, error)) + goto out; + + if (!openat_ignore_enoent (remote_dfd, rev, &ret_fd, error)) + goto out; + + if (ret_fd != -1) break; } ret = TRUE; - ot_transfer_out_value (out_file, &ret_file); + *out_fd = ret_fd; ret_fd = -1; out: return ret; } @@ -210,9 +233,8 @@ resolve_refspec (OstreeRepo *self, { gboolean ret = FALSE; __attribute__((unused)) GCancellable *cancellable = NULL; - GError *temp_error = NULL; g_autofree char *ret_rev = NULL; - g_autoptr(GFile) child = NULL; + glnx_fd_close int target_fd = -1; g_return_val_if_fail (ref != NULL, FALSE); @@ -223,40 +245,42 @@ resolve_refspec (OstreeRepo *self, } else if (remote != NULL) { - child = ot_gfile_resolve_path_printf (self->remote_heads_dir, "%s/%s", - remote, ref); - if (!g_file_query_exists (child, NULL)) - g_clear_object (&child); + const char *remote_ref = glnx_strjoina ("refs/remotes/", remote, "/", ref); + + if (!openat_ignore_enoent (self->repo_dir_fd, remote_ref, &target_fd, error)) + goto out; } else { - child = g_file_resolve_relative_path (self->local_heads_dir, ref); + const char *local_ref = glnx_strjoina ("refs/heads/", ref); - if (!g_file_query_exists (child, NULL)) + if (!openat_ignore_enoent (self->repo_dir_fd, local_ref, &target_fd, error)) + goto out; + + if (target_fd == -1) { - g_clear_object (&child); + local_ref = glnx_strjoina ("refs/remotes/", ref); - child = g_file_resolve_relative_path (self->remote_heads_dir, ref); + if (!openat_ignore_enoent (self->repo_dir_fd, local_ref, &target_fd, error)) + goto out; - if (!g_file_query_exists (child, NULL)) + if (target_fd == -1) { - g_clear_object (&child); - - if (!find_ref_in_remotes (self, ref, &child, error)) + if (!find_ref_in_remotes (self, ref, &target_fd, error)) goto out; } } } - if (child) + if (target_fd != -1) { - if ((ret_rev = gs_file_load_contents_utf8 (child, NULL, &temp_error)) == NULL) + ret_rev = glnx_fd_readall_utf8 (target_fd, NULL, NULL, error); + if (!ret_rev) { - g_propagate_error (error, temp_error); - g_prefix_error (error, "Couldn't open ref '%s': ", gs_file_get_path_cached (child)); + g_prefix_error (error, "Couldn't open ref '%s': ", ref); goto out; } - + g_strchomp (ret_rev); if (!ostree_validate_checksum_string (ret_rev, error)) goto out; @@ -433,43 +457,50 @@ ostree_repo_resolve_rev (OstreeRepo *self, static gboolean enumerate_refs_recurse (OstreeRepo *repo, const char *remote, - GFile *base, - GFile *dir, + int base_dfd, + GString *base_path, + int child_dfd, + const char *path, GHashTable *refs, GCancellable *cancellable, GError **error) { gboolean ret = FALSE; - g_autoptr(GFileEnumerator) enumerator = NULL; + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; - enumerator = g_file_enumerate_children (dir, OSTREE_GIO_FAST_QUERYINFO, - G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - cancellable, error); - if (!enumerator) + if (!glnx_dirfd_iterator_init_at (child_dfd, path, FALSE, &dfd_iter, error)) goto out; while (TRUE) { - GFileInfo *file_info = NULL; - GFile *child = NULL; + guint len = base_path->len; + struct dirent *dent = NULL; - if (!gs_file_enumerator_iterate (enumerator, &file_info, &child, - NULL, error)) + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) goto out; - if (file_info == NULL) + if (dent == NULL) break; - if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_DIRECTORY) + g_string_append (base_path, dent->d_name); + + if (dent->d_type == DT_DIR) { - if (!enumerate_refs_recurse (repo, remote, base, child, refs, cancellable, error)) + g_string_append_c (base_path, '/'); + + if (!enumerate_refs_recurse (repo, remote, base_dfd, base_path, + dfd_iter.fd, dent->d_name, + refs, cancellable, error)) goto out; + } - else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR) + else if (dent->d_type == DT_REG) { - if (!add_ref_to_set (remote, base, child, refs, + if (!add_ref_to_set (remote, base_dfd, base_path->str, refs, cancellable, error)) goto out; } + + g_string_truncate (base_path, len); } ret = TRUE; @@ -505,35 +536,55 @@ ostree_repo_list_refs (OstreeRepo *self, if (refspec_prefix) { - g_autoptr(GFile) dir = NULL; - g_autoptr(GFile) child = NULL; - g_autoptr(GFileInfo) info = NULL; + struct stat stbuf; + const char *prefix_path; + const char *path; if (!ostree_parse_refspec (refspec_prefix, &remote, &ref_prefix, error)) goto out; if (remote) - dir = g_file_get_child (self->remote_heads_dir, remote); - else - dir = g_object_ref (self->local_heads_dir); - - child = g_file_resolve_relative_path (dir, ref_prefix); - if (!ot_gfile_query_info_allow_noent (child, OSTREE_GIO_FAST_QUERYINFO, 0, - &info, cancellable, error)) - goto out; - - if (info) { - if (g_file_info_get_file_type (info) == G_FILE_TYPE_DIRECTORY) + prefix_path = glnx_strjoina ("refs/remotes/", remote, "/"); + path = glnx_strjoina (prefix_path, ref_prefix); + } + else + { + prefix_path = "refs/heads/"; + path = glnx_strjoina (prefix_path, ref_prefix); + } + + if (fstatat (self->repo_dir_fd, path, &stbuf, 0) < 0) + { + if (errno != ENOENT) { - if (!enumerate_refs_recurse (self, remote, child, child, - ret_all_refs, - cancellable, error)) + glnx_set_error_from_errno (error); + goto out; + } + } + else + { + if (S_ISDIR (stbuf.st_mode)) + { + glnx_fd_close int base_fd = -1; + g_autoptr(GString) base_path = g_string_new (""); + + if (!glnx_opendirat (self->repo_dir_fd, path, TRUE, &base_fd, error)) + goto out; + + if (!enumerate_refs_recurse (self, remote, base_fd, base_path, + base_fd, ".", + ret_all_refs, cancellable, error)) goto out; } else { - if (!add_ref_to_set (remote, dir, child, ret_all_refs, + glnx_fd_close int prefix_dfd = -1; + + if (!glnx_opendirat (self->repo_dir_fd, prefix_path, TRUE, &prefix_dfd, error)) + goto out; + + if (!add_ref_to_set (remote, prefix_dfd, ref_prefix, ret_all_refs, cancellable, error)) goto out; } @@ -541,31 +592,41 @@ ostree_repo_list_refs (OstreeRepo *self, } else { - g_autoptr(GFileEnumerator) remote_enumerator = NULL; - - if (!enumerate_refs_recurse (self, NULL, self->local_heads_dir, self->local_heads_dir, - ret_all_refs, - cancellable, error)) + g_auto(GLnxDirFdIterator) dfd_iter = { 0, }; + g_autoptr(GString) base_path = g_string_new (""); + glnx_fd_close int refs_heads_dfd = -1; + + if (!glnx_opendirat (self->repo_dir_fd, "refs/heads", TRUE, &refs_heads_dfd, error)) goto out; - remote_enumerator = g_file_enumerate_children (self->remote_heads_dir, OSTREE_GIO_FAST_QUERYINFO, - 0, - cancellable, error); + if (!enumerate_refs_recurse (self, NULL, refs_heads_dfd, base_path, + refs_heads_dfd, ".", + ret_all_refs, cancellable, error)) + goto out; + + g_string_truncate (base_path, 0); + + if (!glnx_dirfd_iterator_init_at (self->repo_dir_fd, "refs/remotes", TRUE, &dfd_iter, error)) + goto out; while (TRUE) { - GFileInfo *info; - GFile *child; - const char *name; + struct dirent *dent; + glnx_fd_close int remote_dfd = -1; - if (!gs_file_enumerator_iterate (remote_enumerator, &info, &child, - cancellable, error)) + if (!glnx_dirfd_iterator_next_dent_ensure_dtype (&dfd_iter, &dent, cancellable, error)) goto out; - if (!info) + if (!dent) break; - name = g_file_info_get_name (info); - if (!enumerate_refs_recurse (self, name, child, child, + if (dent->d_type != DT_DIR) + continue; + + if (!glnx_opendirat (dfd_iter.fd, dent->d_name, TRUE, &remote_dfd, error)) + goto out; + + if (!enumerate_refs_recurse (self, dent->d_name, remote_dfd, base_path, + remote_dfd, ".", ret_all_refs, cancellable, error)) goto out; diff --git a/src/libostree/ostree-repo-static-delta-compilation.c b/src/libostree/ostree-repo-static-delta-compilation.c index 530b3a75..2d02e6a6 100644 --- a/src/libostree/ostree-repo-static-delta-compilation.c +++ b/src/libostree/ostree-repo-static-delta-compilation.c @@ -59,6 +59,7 @@ typedef struct { guint n_rollsum; guint n_bsdiff; guint n_fallback; + gboolean swap_endian; } OstreeStaticDeltaBuilder; typedef enum { @@ -228,7 +229,7 @@ objtype_checksum_array_new (GPtrArray *objects) GVariant *serialized_key = objects->pdata[i]; OstreeObjectType objtype; const char *checksum; - guint8 csum[32]; + guint8 csum[OSTREE_SHA256_DIGEST_LEN]; guint8 objtype_v; ostree_object_name_deserialize (serialized_key, &checksum, &objtype); @@ -678,7 +679,7 @@ process_one_rollsum (OstreeRepo *repo, { gsize mode_offset, xattr_offset, from_csum_offset; gboolean reading_payload = TRUE; - guchar source_csum[32]; + guchar source_csum[OSTREE_SHA256_DIGEST_LEN]; guint i; write_content_mode_xattrs (repo, current_part, content_finfo, content_xattrs, @@ -799,7 +800,7 @@ process_one_bsdiff (OstreeRepo *repo, g_ptr_array_add (current_part->objects, ostree_object_name_serialize (to_checksum, OSTREE_OBJECT_TYPE_FILE)); { gsize mode_offset, xattr_offset; - guchar source_csum[32]; + guchar source_csum[OSTREE_SHA256_DIGEST_LEN]; write_content_mode_xattrs (repo, current_part, content_finfo, content_xattrs, &mode_offset, &xattr_offset); @@ -1191,7 +1192,8 @@ get_fallback_headers (OstreeRepo *self, g_variant_new ("(y@aytt)", objtype, ostree_checksum_to_bytes_v (checksum), - compressed_size, uncompressed_size)); + maybe_swap_endian_u64 (builder->swap_endian, compressed_size), + maybe_swap_endian_u64 (builder->swap_endian, uncompressed_size))); } ret_headers = g_variant_ref_sink (g_variant_builder_end (fallback_builder)); @@ -1228,6 +1230,7 @@ get_fallback_headers (OstreeRepo *self, * - bsdiff-enabled: b: Enable bsdiff compression. Default TRUE. * - inline-parts: b: Put part data in header, to get a single file delta. Default FALSE. * - verbose: b: Print diagnostic messages. Default FALSE. + * - endianness: b: Deltas use host byte order by default; this option allows choosing (G_BIG_ENDIAN or G_LITTLE_ENDIAN) * - filename: ay: Save delta superblock to this filename, and parts in the same directory. Default saves to repository. */ gboolean @@ -1262,6 +1265,7 @@ ostree_repo_static_delta_generate (OstreeRepo *self, g_autoptr(GVariant) fallback_headers = NULL; g_autoptr(GVariant) detached = NULL; gboolean inline_parts; + guint endianness = G_BYTE_ORDER; g_autoptr(GFile) tmp_dir = NULL; builder.parts = g_ptr_array_new_with_free_func ((GDestroyNotify)ostree_static_delta_part_builder_unref); builder.fallback_objects = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref); @@ -1277,6 +1281,11 @@ ostree_repo_static_delta_generate (OstreeRepo *self, max_chunk_size = 32; builder.max_chunk_size_bytes = ((guint64)max_chunk_size) * 1000 * 1000; + (void) g_variant_lookup (params, "endianness", "u", &endianness); + g_return_val_if_fail (endianness == G_BIG_ENDIAN || endianness == G_LITTLE_ENDIAN, FALSE); + + builder.swap_endian = endianness != G_BYTE_ORDER; + { gboolean use_bsdiff; if (!g_variant_lookup (params, "bsdiff-enabled", "b", &use_bsdiff)) use_bsdiff = TRUE; @@ -1306,6 +1315,10 @@ ostree_repo_static_delta_generate (OstreeRepo *self, cancellable, error)) goto out; + /* NOTE: Add user-supplied metadata first. This is used by at least + * xdg-app as a way to provide MIME content sniffing, since the + * metadata appears first in the file. + */ g_variant_builder_init (&metadata_builder, G_VARIANT_TYPE ("a{sv}")); if (metadata != NULL) { @@ -1320,6 +1333,22 @@ ostree_repo_static_delta_generate (OstreeRepo *self, } } + { guint8 endianness_char; + + switch (endianness) + { + case G_LITTLE_ENDIAN: + endianness_char = 'l'; + break; + case G_BIG_ENDIAN: + endianness_char = 'B'; + break; + default: + g_assert_not_reached (); + } + g_variant_builder_add (&metadata_builder, "{sv}", "ostree.endianness", g_variant_new_byte (endianness_char)); + } + if (opt_filename) { g_autoptr(GFile) f = g_file_new_for_path (opt_filename); @@ -1411,13 +1440,13 @@ ostree_repo_static_delta_generate (OstreeRepo *self, cancellable, error)) goto out; - checksum_bytes = g_bytes_new (part_checksum, 32); + checksum_bytes = g_bytes_new (part_checksum, OSTREE_SHA256_DIGEST_LEN); objtype_checksum_array = objtype_checksum_array_new (part_builder->objects); delta_part_header = g_variant_new ("(u@aytt@ay)", - OSTREE_DELTAPART_VERSION, + maybe_swap_endian_u32 (builder.swap_endian, OSTREE_DELTAPART_VERSION), ot_gvariant_new_ay_bytes (checksum_bytes), - (guint64) g_variant_get_size (delta_part), - part_builder->uncompressed_size, + maybe_swap_endian_u64 (builder.swap_endian, (guint64) g_variant_get_size (delta_part)), + maybe_swap_endian_u64 (builder.swap_endian, part_builder->uncompressed_size), ot_gvariant_new_ay_bytes (objtype_checksum_array)); g_variant_builder_add_value (part_headers, g_variant_ref (delta_part_header)); diff --git a/src/libostree/ostree-repo-static-delta-core.c b/src/libostree/ostree-repo-static-delta-core.c index e3490fd6..d84f0019 100644 --- a/src/libostree/ostree-repo-static-delta-core.c +++ b/src/libostree/ostree-repo-static-delta-core.c @@ -20,8 +20,14 @@ #include "config.h" +#include +#include +#include #include "ostree-core-private.h" #include "ostree-repo-private.h" +#include "ostree-lzma-decompressor.h" +#include "ostree-cmdprivate.h" +#include "ostree-checksum-input-stream.h" #include "ostree-repo-static-delta-private.h" #include "otutil.h" @@ -131,7 +137,7 @@ ostree_repo_list_static_delta_names (OstreeRepo *self, g_autofree char *buf = g_strconcat (name1, name2, NULL); GString *out = g_string_new (""); char checksum[65]; - guchar csum[32]; + guchar csum[OSTREE_SHA256_DIGEST_LEN]; const char *dash = strchr (buf, '-'); ostree_checksum_b64_inplace_to_bytes (buf, csum); @@ -225,31 +231,49 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, { gboolean ret = FALSE; guint i, n; - g_autoptr(GFile) meta_file = NULL; - g_autoptr(GFile) dir = NULL; + const char *dir_or_file_path = NULL; + glnx_fd_close int meta_fd = -1; + glnx_fd_close int dfd = -1; g_autoptr(GVariant) meta = NULL; g_autoptr(GVariant) headers = NULL; g_autoptr(GVariant) metadata = NULL; g_autoptr(GVariant) fallback = NULL; g_autofree char *to_checksum = NULL; g_autofree char *from_checksum = NULL; - GFileType file_type; + g_autofree char *basename = NULL; + dir_or_file_path = gs_file_get_path_cached (dir_or_file); - file_type = g_file_query_file_type (dir_or_file, 0, cancellable); - if (file_type == G_FILE_TYPE_DIRECTORY) + /* First, try opening it as a directory */ + dfd = glnx_opendirat_with_errno (AT_FDCWD, dir_or_file_path, TRUE); + if (dfd < 0) { - dir = g_object_ref (dir_or_file); - meta_file = g_file_get_child (dir, "superblock"); + if (errno != ENOTDIR) + { + glnx_set_error_from_errno (error); + goto out; + } + else + { + g_autofree char *dir = dirname (g_strdup (dir_or_file_path)); + basename = g_path_get_basename (dir_or_file_path); + + if (!glnx_opendirat (AT_FDCWD, dir, TRUE, &dfd, error)) + goto out; + } } else - { - meta_file = g_object_ref (dir_or_file); - dir = g_file_get_parent (meta_file); - } + basename = g_strdup ("superblock"); - if (!ot_util_variant_map (meta_file, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), - FALSE, &meta, error)) + meta_fd = openat (dfd, basename, O_RDONLY | O_CLOEXEC); + if (meta_fd < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + if (!ot_util_variant_map_fd (meta_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT), + FALSE, &meta, error)) goto out; /* Parsing OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT */ @@ -329,14 +353,18 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, guint64 size; guint64 usize; const guchar *csum; + char checksum[65]; gboolean have_all; + g_autoptr(GInputStream) part_in = NULL; g_autoptr(GBytes) delta_data = NULL; - g_autoptr(GVariant) part_data = NULL; + g_autoptr(GVariant) inline_part_data = NULL; g_autoptr(GVariant) header = NULL; g_autoptr(GVariant) csum_v = NULL; g_autoptr(GVariant) objects = NULL; - g_autoptr(GBytes) bytes = NULL; + g_autoptr(GVariant) part = NULL; g_autofree char *deltapart_path = NULL; + OstreeStaticDeltaOpenFlags delta_open_flags = + skip_validation ? OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM : 0; header = g_variant_get_child_value (headers, i); g_variant_get (header, "(u@aytt@ay)", &version, &csum_v, &size, &usize, &objects); @@ -361,41 +389,57 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, csum = ostree_checksum_bytes_peek_validate (csum_v, error); if (!csum) goto out; + ostree_checksum_inplace_from_bytes (csum, checksum); deltapart_path = _ostree_get_relative_static_delta_part_path (from_checksum, to_checksum, i); - part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)")); - if (part_data) + inline_part_data = g_variant_lookup_value (metadata, deltapart_path, G_VARIANT_TYPE("(yay)")); + if (inline_part_data) { - bytes = g_variant_get_data_as_bytes (part_data); + g_autoptr(GBytes) inline_part_bytes = g_variant_get_data_as_bytes (inline_part_data); + part_in = g_memory_input_stream_new_from_bytes (inline_part_bytes); + + /* For inline parts, we don't checksum, because it's + * included with the metadata, so we're not trying to + * protect against MITM or such. Non-security related + * checksums should be done at the underlying storage layer. + */ + delta_open_flags |= OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM; + + if (!_ostree_static_delta_part_open (part_in, inline_part_bytes, + delta_open_flags, + NULL, + &part, + cancellable, error)) + goto out; } else { - g_autoptr(GFile) part_path = ot_gfile_resolve_path_printf (dir, "%u", i); - GMappedFile *mfile = gs_file_map_noatime (part_path, cancellable, error); - if (!mfile) - goto out; + g_autofree char *relpath = g_strdup_printf ("%u", i); /* TODO avoid malloc here */ + glnx_fd_close int part_fd = openat (dfd, relpath, O_RDONLY | O_CLOEXEC); + if (part_fd < 0) + { + glnx_set_error_from_errno (error); + g_prefix_error (error, "Opening deltapart '%s': ", deltapart_path); + goto out; + } - bytes = g_mapped_file_get_bytes (mfile); - g_mapped_file_unref (mfile); - } + part_in = g_unix_input_stream_new (part_fd, FALSE); - if (!skip_validation) - { - g_autoptr(GInputStream) in = g_memory_input_stream_new_from_bytes (bytes); - - g_autofree char *expected_checksum = ostree_checksum_from_bytes (csum); - if (!_ostree_static_delta_part_validate (self, in, i, - expected_checksum, - cancellable, error)) + if (!_ostree_static_delta_part_open (part_in, NULL, + delta_open_flags, + checksum, + &part, + cancellable, error)) goto out; } - if (!_ostree_static_delta_part_execute (self, objects, bytes, skip_validation, + if (!_ostree_static_delta_part_execute (self, objects, part, skip_validation, + FALSE, NULL, cancellable, error)) { - g_prefix_error (error, "executing delta part %i: ", i); + g_prefix_error (error, "Executing delta part %i: ", i); goto out; } } @@ -404,3 +448,462 @@ ostree_repo_static_delta_execute_offline (OstreeRepo *self, out: return ret; } + +gboolean +_ostree_static_delta_part_open (GInputStream *part_in, + GBytes *inline_part_bytes, + OstreeStaticDeltaOpenFlags flags, + const char *expected_checksum, + GVariant **out_part, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + const gboolean trusted = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED) > 0; + const gboolean skip_checksum = (flags & OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM) > 0; + gsize bytes_read; + guint8 comptype; + g_autoptr(GChecksum) checksum = NULL; + g_autoptr(GInputStream) checksum_in = NULL; + g_autoptr(GVariant) ret_part = NULL; + GInputStream *source_in; + + /* We either take a fd or a GBytes reference */ + g_return_val_if_fail (G_IS_FILE_DESCRIPTOR_BASED (part_in) || inline_part_bytes != NULL, FALSE); + g_return_val_if_fail (skip_checksum || expected_checksum != NULL, FALSE); + + if (!skip_checksum) + { + checksum = g_checksum_new (G_CHECKSUM_SHA256); + checksum_in = (GInputStream*)ostree_checksum_input_stream_new (part_in, checksum); + source_in = checksum_in; + } + else + { + source_in = part_in; + } + + { guint8 buf[1]; + /* First byte is compression type */ + if (!g_input_stream_read_all (source_in, buf, sizeof(buf), &bytes_read, + cancellable, error)) + { + g_prefix_error (error, "Reading initial compression flag byte: "); + goto out; + } + comptype = buf[0]; + } + + switch (comptype) + { + case 0: + if (!inline_part_bytes) + { + int part_fd = g_file_descriptor_based_get_fd ((GFileDescriptorBased*)part_in); + + /* No compression, no checksums - a fast path */ + if (!ot_util_variant_map_fd (part_fd, 1, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), + trusted, &ret_part, error)) + goto out; + } + else + { + g_autoptr(GBytes) content_bytes = g_bytes_new_from_bytes (inline_part_bytes, 1, + g_bytes_get_size (inline_part_bytes) - 1); + ret_part = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), + content_bytes, trusted); + } + + if (!skip_checksum) + g_checksum_update (checksum, g_variant_get_data (ret_part), + g_variant_get_size (ret_part)); + + break; + case 'x': + { + g_autofree char *tmppath = g_strdup ("/var/tmp/ostree-delta-XXXXXX"); + g_autoptr(GConverter) decomp = (GConverter*) _ostree_lzma_decompressor_new (); + g_autoptr(GInputStream) convin = g_converter_input_stream_new (source_in, decomp); + g_autoptr(GOutputStream) unpacked_out = NULL; + glnx_fd_close int unpacked_fd = -1; + gssize n_bytes_written; + + unpacked_fd = g_mkstemp_full (tmppath, O_RDWR | O_CLOEXEC, 0640); + if (unpacked_fd < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + /* Now make it autocleanup on process exit - in the future, we + * should consider caching unpacked deltas as well. + */ + if (unlink (tmppath) < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + unpacked_out = g_unix_output_stream_new (unpacked_fd, FALSE); + + n_bytes_written = g_output_stream_splice (unpacked_out, convin, + G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, + cancellable, error); + if (n_bytes_written < 0) + goto out; + + if (!ot_util_variant_map_fd (unpacked_fd, 0, G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), + trusted, &ret_part, error)) + goto out; + } + break; + default: + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid compression type '%u'", comptype); + goto out; + } + + if (checksum) + { + const char *actual_checksum = g_checksum_get_string (checksum); + g_assert (expected_checksum != NULL); + if (strcmp (actual_checksum, expected_checksum) != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Checksum mismatch in static delta part; expected=%s actual=%s", + expected_checksum, actual_checksum); + goto out; + } + } + + ret = TRUE; + *out_part = g_steal_pointer (&ret_part); + out: + return ret; +} + +/* + * Displaying static delta parts + */ + +static gboolean +show_one_part (OstreeRepo *self, + gboolean swap_endian, + const char *from, + const char *to, + GVariant *meta_entries, + guint i, + guint64 *total_size_ref, + guint64 *total_usize_ref, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + guint32 version; + guint64 size, usize; + g_autoptr(GVariant) objects = NULL; + g_autoptr(GInputStream) part_in = NULL; + g_autoptr(GVariant) part = NULL; + g_autofree char *part_path = _ostree_get_relative_static_delta_part_path (from, to, i); + gint part_fd = -1; + + g_variant_get_child (meta_entries, i, "(u@aytt@ay)", &version, NULL, &size, &usize, &objects); + size = maybe_swap_endian_u64 (swap_endian, size); + usize = maybe_swap_endian_u64 (swap_endian, usize); + *total_size_ref += size; + *total_usize_ref += usize; + g_print ("PartMeta%u: nobjects=%u size=%" G_GUINT64_FORMAT " usize=%" G_GUINT64_FORMAT "\n", + i, (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN), size, usize); + + part_fd = openat (self->repo_dir_fd, part_path, O_RDONLY | O_CLOEXEC); + if (part_fd < 0) + { + glnx_set_error_from_errno (error); + goto out; + } + + part_in = g_unix_input_stream_new (part_fd, FALSE); + + if (!_ostree_static_delta_part_open (part_in, NULL, + OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM, + NULL, + &part, + cancellable, error)) + goto out; + + { g_autoptr(GVariant) modes = NULL; + g_autoptr(GVariant) xattrs = NULL; + g_autoptr(GVariant) blob = NULL; + g_autoptr(GVariant) ops = NULL; + OstreeDeltaExecuteStats stats = { { 0, }, }; + + g_variant_get (part, "(@a(uuu)@aa(ayay)@ay@ay)", + &modes, &xattrs, &blob, &ops); + + g_print ("PartPayload%u: nmodes=%" G_GUINT64_FORMAT + " nxattrs=%" G_GUINT64_FORMAT + " blobsize=%" G_GUINT64_FORMAT + " opsize=%" G_GUINT64_FORMAT + "\n", + i, + (guint64)g_variant_n_children (modes), + (guint64)g_variant_n_children (xattrs), + (guint64)g_variant_n_children (blob), + (guint64)g_variant_n_children (ops)); + + if (!_ostree_static_delta_part_execute (self, objects, + part, TRUE, TRUE, + &stats, cancellable, error)) + goto out; + + { const guint *n_ops = stats.n_ops_executed; + g_print ("PartPayloadOps%u: openspliceclose=%u open=%u write=%u setread=%u " + "unsetread=%u close=%u bspatch=%u\n", + i, n_ops[0], n_ops[1], n_ops[2], n_ops[3], n_ops[4], n_ops[5], n_ops[6]); + } + } + + ret = TRUE; + out: + return ret; +} + +OstreeDeltaEndianness +_ostree_delta_get_endianness (GVariant *superblock, + gboolean *out_was_heuristic) +{ + guint8 endianness_char; + g_autoptr(GVariant) delta_meta = NULL; + g_autoptr(GVariantDict) delta_metadict = NULL; + guint64 total_size = 0; + guint64 total_usize = 0; + guint total_objects = 0; + + delta_meta = g_variant_get_child_value (superblock, 0); + delta_metadict = g_variant_dict_new (delta_meta); + + if (out_was_heuristic) + *out_was_heuristic = FALSE; + + if (g_variant_dict_lookup (delta_metadict, "ostree.endianness", "y", &endianness_char)) + { + switch (endianness_char) + { + case 'l': + return OSTREE_DELTA_ENDIAN_LITTLE; + case 'B': + return OSTREE_DELTA_ENDIAN_BIG; + default: + return OSTREE_DELTA_ENDIAN_INVALID; + } + } + + if (out_was_heuristic) + *out_was_heuristic = TRUE; + + { g_autoptr(GVariant) meta_entries = NULL; + guint n_parts; + guint i; + gboolean is_byteswapped = FALSE; + + g_variant_get_child (superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); + n_parts = g_variant_n_children (meta_entries); + + for (i = 0; i < n_parts; i++) + { + g_autoptr(GVariant) objects = NULL; + guint64 size, usize; + guint n_objects; + + g_variant_get_child (meta_entries, i, "(u@aytt@ay)", NULL, NULL, &size, &usize, &objects); + n_objects = (guint)(g_variant_get_size (objects) / OSTREE_STATIC_DELTA_OBJTYPE_CSUM_LEN); + + total_objects += n_objects; + total_size += size; + total_usize += usize; + + if (size > usize) + { + double ratio = ((double)size)/((double)usize); + + /* This should really never happen where compressing things makes it more than 50% bigger. + */ + if (ratio > 1.2) + { + is_byteswapped = TRUE; + break; + } + } + } + + if (!is_byteswapped) + { + /* If the average object size is greater than 4GiB, let's assume + * we're dealing with opposite endianness. I'm fairly confident + * no one is going to be shipping peta- or exa- byte size ostree + * deltas, period. Past the gigabyte scale you really want + * bittorrent or something. + */ + if ((total_size / total_objects) > G_MAXUINT32) + { + is_byteswapped = TRUE; + } + } + + if (is_byteswapped) + { + switch (G_BYTE_ORDER) + { + case G_BIG_ENDIAN: + return OSTREE_DELTA_ENDIAN_LITTLE; + case G_LITTLE_ENDIAN: + return OSTREE_DELTA_ENDIAN_BIG; + default: + g_assert_not_reached (); + } + } + + return G_BYTE_ORDER; + } +} + +gboolean +_ostree_delta_needs_byteswap (GVariant *superblock) +{ + switch (_ostree_delta_get_endianness (superblock, NULL)) + { + case OSTREE_DELTA_ENDIAN_BIG: + return G_BYTE_ORDER == G_LITTLE_ENDIAN; + case OSTREE_DELTA_ENDIAN_LITTLE: + return G_BYTE_ORDER == G_BIG_ENDIAN; + default: + return FALSE; + } +} + +gboolean +_ostree_repo_static_delta_dump (OstreeRepo *self, + const char *delta_id, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + g_autofree char *from = NULL; + g_autofree char *to = NULL; + g_autofree char *superblock_path = NULL; + glnx_fd_close int superblock_fd = -1; + g_autoptr(GVariant) delta_superblock = NULL; + guint64 total_size = 0, total_usize = 0; + guint64 total_fallback_size = 0, total_fallback_usize = 0; + guint i; + OstreeDeltaEndianness endianness; + gboolean swap_endian = FALSE; + + _ostree_parse_delta_name (delta_id, &from, &to); + superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to); + + if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path, + (GVariantType*)OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT, + TRUE, &delta_superblock, error)) + goto out; + + g_print ("%s\n", g_variant_print (delta_superblock, 1)); + + g_print ("Delta: %s\n", delta_id); + { const char *endianness_description; + gboolean was_heuristic; + + endianness = _ostree_delta_get_endianness (delta_superblock, &was_heuristic); + + switch (endianness) + { + case OSTREE_DELTA_ENDIAN_BIG: + if (was_heuristic) + endianness_description = "big (heuristic)"; + else + endianness_description = "big"; + if (G_BYTE_ORDER == G_LITTLE_ENDIAN) + swap_endian = TRUE; + break; + case OSTREE_DELTA_ENDIAN_LITTLE: + if (was_heuristic) + endianness_description = "little (heuristic)"; + else + endianness_description = "little"; + if (G_BYTE_ORDER == G_BIG_ENDIAN) + swap_endian = TRUE; + break; + case OSTREE_DELTA_ENDIAN_INVALID: + endianness_description = "invalid"; + break; + default: + g_assert_not_reached (); + } + + g_print ("Endianness: %s\n", endianness_description); + } + { guint64 ts; + g_variant_get_child (delta_superblock, 1, "t", &ts); + g_print ("Timestamp: %" G_GUINT64_FORMAT "\n", GUINT64_FROM_BE (ts)); + } + { g_autoptr(GVariant) recurse = NULL; + g_variant_get_child (delta_superblock, 5, "@ay", &recurse); + g_print ("Number of parents: %u\n", (guint)(g_variant_get_size (recurse) / (OSTREE_SHA256_DIGEST_LEN * 2))); + } + { g_autoptr(GVariant) fallback = NULL; + guint n_fallback; + + g_variant_get_child (delta_superblock, 7, "@a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT, &fallback); + n_fallback = g_variant_n_children (fallback); + + g_print ("Number of fallback entries: %u\n", n_fallback); + + for (i = 0; i < n_fallback; i++) + { + guint64 size, usize; + g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize); + size = maybe_swap_endian_u64 (swap_endian, size); + usize = maybe_swap_endian_u64 (swap_endian, usize); + total_fallback_size += size; + total_fallback_usize += usize; + } + { g_autofree char *sizestr = g_format_size (total_fallback_size); + g_autofree char *usizestr = g_format_size (total_fallback_usize); + g_print ("Total Fallback Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_size, sizestr); + g_print ("Total Fallback Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_fallback_usize, usizestr); + } + } + { g_autoptr(GVariant) meta_entries = NULL; + guint n_parts; + + g_variant_get_child (delta_superblock, 6, "@a" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT, &meta_entries); + n_parts = g_variant_n_children (meta_entries); + g_print ("Number of parts: %u\n", n_parts); + + for (i = 0; i < n_parts; i++) + { + if (!show_one_part (self, swap_endian, from, to, meta_entries, i, + &total_size, &total_usize, + cancellable, error)) + goto out; + } + } + + { g_autofree char *sizestr = g_format_size (total_size); + g_autofree char *usizestr = g_format_size (total_usize); + g_print ("Total Part Size: %" G_GUINT64_FORMAT " (%s)\n", total_size, sizestr); + g_print ("Total Part Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", total_usize, usizestr); + } + { guint64 overall_size = total_size + total_fallback_size; + guint64 overall_usize = total_usize + total_fallback_usize; + g_autofree char *sizestr = g_format_size (overall_size); + g_autofree char *usizestr = g_format_size (overall_usize); + g_print ("Total Size: %" G_GUINT64_FORMAT " (%s)\n", overall_size, sizestr); + g_print ("Total Uncompressed Size: %" G_GUINT64_FORMAT " (%s)\n", overall_usize, usizestr); + } + + ret = TRUE; + out: + return ret; +} diff --git a/src/libostree/ostree-repo-static-delta-private.h b/src/libostree/ostree-repo-static-delta-private.h index de32ec65..41ddad48 100644 --- a/src/libostree/ostree-repo-static-delta-private.h +++ b/src/libostree/ostree-repo-static-delta-private.h @@ -46,10 +46,10 @@ G_BEGIN_DECLS /** * OSTREE_STATIC_DELTA_META_ENTRY_FORMAT: * - * u: version + * u: version (non-canonical endian) * ay checksum - * guint64 size: Total size of delta (sum of parts) - * guint64 usize: Uncompressed size of resulting objects on disk + * guint64 size: Total size of delta (sum of parts) (non-canonical endian) + * guint64 usize: Uncompressed size of resulting objects on disk (non-canonical endian) * ARRAY[(guint8 objtype, csum object)] * * The checksum is of the delta payload, and each entry in the array @@ -64,8 +64,8 @@ G_BEGIN_DECLS * * y: objtype * ay: checksum - * t: compressed size - * t: uncompressed size + * t: compressed size (non-canonical endian) + * t: uncompressed size (non-canonical endian) * * Object to fetch invididually; includes compressed/uncompressed size. */ @@ -79,7 +79,7 @@ G_BEGIN_DECLS * * delta-descriptor: * metadata: a{sv} - * t: timestamp + * t: timestamp (big endian) * from: ay checksum * to: ay checksum * commit: new commit object @@ -103,38 +103,11 @@ G_BEGIN_DECLS */ #define OSTREE_STATIC_DELTA_SUPERBLOCK_FORMAT "(a{sv}tayay" OSTREE_COMMIT_GVARIANT_STRING "aya" OSTREE_STATIC_DELTA_META_ENTRY_FORMAT "a" OSTREE_STATIC_DELTA_FALLBACK_FORMAT ")" -gboolean _ostree_static_delta_part_validate (OstreeRepo *repo, - GInputStream *in, - guint part_offset, - const char *expected_checksum, - GCancellable *cancellable, - GError **error); - -gboolean _ostree_static_delta_part_execute (OstreeRepo *repo, - GVariant *header, - GBytes *partdata, - gboolean trusted, - GCancellable *cancellable, - GError **error); - -gboolean _ostree_static_delta_part_execute_raw (OstreeRepo *repo, - GVariant *header, - GVariant *part, - gboolean trusted, - GCancellable *cancellable, - GError **error); - -void _ostree_static_delta_part_execute_async (OstreeRepo *repo, - GVariant *header, - GBytes *partdata, - gboolean trusted, - GCancellable *cancellable, - GAsyncReadyCallback callback, - gpointer user_data); - -gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo, - GAsyncResult *result, - GError **error); +typedef enum { + OSTREE_STATIC_DELTA_OPEN_FLAGS_NONE = 0, + OSTREE_STATIC_DELTA_OPEN_FLAGS_SKIP_CHECKSUM = (1 << 0), + OSTREE_STATIC_DELTA_OPEN_FLAGS_VARIANT_TRUSTED = (1 << 1) +} OstreeStaticDeltaOpenFlags; typedef enum { OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE = 'S', @@ -145,6 +118,46 @@ typedef enum { OSTREE_STATIC_DELTA_OP_CLOSE = 'c', OSTREE_STATIC_DELTA_OP_BSPATCH = 'B' } OstreeStaticDeltaOpCode; +#define OSTREE_STATIC_DELTA_N_OPS 7 + +gboolean +_ostree_static_delta_part_open (GInputStream *part_in, + GBytes *inline_part_bytes, + OstreeStaticDeltaOpenFlags flags, + const char *expected_checksum, + GVariant **out_part, + GCancellable *cancellable, + GError **error); + +gboolean _ostree_static_delta_dump (OstreeRepo *repo, + const char *delta_id, + GCancellable *cancellable, + GError **error); + +typedef struct { + guint n_ops_executed[OSTREE_STATIC_DELTA_N_OPS]; +} OstreeDeltaExecuteStats; + +gboolean _ostree_static_delta_part_execute (OstreeRepo *repo, + GVariant *header, + GVariant *part_payload, + gboolean trusted, + gboolean stats_only, + OstreeDeltaExecuteStats *stats, + GCancellable *cancellable, + GError **error); + +void _ostree_static_delta_part_execute_async (OstreeRepo *repo, + GVariant *header, + GVariant *part_payload, + gboolean trusted, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); + +gboolean _ostree_static_delta_part_execute_finish (OstreeRepo *repo, + GAsyncResult *result, + GError **error); gboolean _ostree_static_delta_parse_checksum_array (GVariant *array, @@ -177,4 +190,43 @@ _ostree_delta_compute_similar_objects (OstreeRepo *repo, GCancellable *cancellable, GError **error); +gboolean +_ostree_repo_static_delta_dump (OstreeRepo *repo, + const char *delta_id, + GCancellable *cancellable, + GError **error); + +/* Used for static deltas which due to a historical mistake are + * inconsistent endian. + * + * https://bugzilla.gnome.org/show_bug.cgi?id=762515 + */ +static inline guint32 +maybe_swap_endian_u32 (gboolean swap, + guint32 v) +{ + if (!swap) + return v; + return GUINT32_SWAP_LE_BE (v); +} + +static inline guint64 +maybe_swap_endian_u64 (gboolean swap, + guint64 v) +{ + if (!swap) + return v; + return GUINT64_SWAP_LE_BE (v); +} + +typedef enum { + OSTREE_DELTA_ENDIAN_BIG, + OSTREE_DELTA_ENDIAN_LITTLE, + OSTREE_DELTA_ENDIAN_INVALID +} OstreeDeltaEndianness; + +OstreeDeltaEndianness _ostree_delta_get_endianness (GVariant *superblock, gboolean *out_was_heuristic); + +gboolean _ostree_delta_needs_byteswap (GVariant *superblock); + G_END_DECLS diff --git a/src/libostree/ostree-repo-static-delta-processing.c b/src/libostree/ostree-repo-static-delta-processing.c index 10e533c6..b44314b6 100644 --- a/src/libostree/ostree-repo-static-delta-processing.c +++ b/src/libostree/ostree-repo-static-delta-processing.c @@ -40,6 +40,7 @@ G_STATIC_ASSERT (sizeof (guint) >= sizeof (guint32)); typedef struct { gboolean trusted; + gboolean stats_only; OstreeRepo *repo; guint checksum_index; const guint8 *checksums; @@ -149,43 +150,39 @@ open_output_target (StaticDeltaExecutionState *state, return ret; } -gboolean -_ostree_static_delta_part_validate (OstreeRepo *repo, - GInputStream *in, - guint part_offset, - const char *expected_checksum, - GCancellable *cancellable, - GError **error) +static guint +delta_opcode_index (OstreeStaticDeltaOpCode op) { - gboolean ret = FALSE; - g_autofree guchar *actual_checksum_bytes = NULL; - g_autofree char *actual_checksum = NULL; - - if (!ot_gio_checksum_stream (in, &actual_checksum_bytes, - cancellable, error)) - goto out; - - actual_checksum = ostree_checksum_from_bytes (actual_checksum_bytes); - if (strcmp (actual_checksum, expected_checksum) != 0) + switch (op) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Checksum mismatch in static delta part %u; expected=%s actual=%s", - part_offset, expected_checksum, actual_checksum); - goto out; + case OSTREE_STATIC_DELTA_OP_OPEN_SPLICE_AND_CLOSE: + return 0; + case OSTREE_STATIC_DELTA_OP_OPEN: + return 1; + case OSTREE_STATIC_DELTA_OP_WRITE: + return 2; + case OSTREE_STATIC_DELTA_OP_SET_READ_SOURCE: + return 3; + case OSTREE_STATIC_DELTA_OP_UNSET_READ_SOURCE: + return 4; + case OSTREE_STATIC_DELTA_OP_CLOSE: + return 5; + case OSTREE_STATIC_DELTA_OP_BSPATCH: + return 6; + default: + g_assert_not_reached (); } - - ret = TRUE; - out: - return ret; } gboolean -_ostree_static_delta_part_execute_raw (OstreeRepo *repo, - GVariant *objects, - GVariant *part, - gboolean trusted, - GCancellable *cancellable, - GError **error) +_ostree_static_delta_part_execute (OstreeRepo *repo, + GVariant *objects, + GVariant *part, + gboolean trusted, + gboolean stats_only, + OstreeDeltaExecuteStats *stats, + GCancellable *cancellable, + GError **error) { gboolean ret = FALSE; guint8 *checksums_data; @@ -201,6 +198,7 @@ _ostree_static_delta_part_execute_raw (OstreeRepo *repo, state->repo = repo; state->async_error = error; state->trusted = trusted; + state->stats_only = stats_only; if (!_ostree_static_delta_parse_checksum_array (objects, &checksums_data, @@ -270,6 +268,8 @@ _ostree_static_delta_part_execute_raw (OstreeRepo *repo, } n_executed++; + if (stats) + stats->n_ops_executed[delta_opcode_index(opcode)]++; } if (state->caught_error) @@ -280,99 +280,10 @@ _ostree_static_delta_part_execute_raw (OstreeRepo *repo, return ret; } -static gboolean -decompress_all (GConverter *converter, - GBytes *data, - GBytes **out_uncompressed, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - g_autoptr(GMemoryInputStream) memin = (GMemoryInputStream*)g_memory_input_stream_new_from_bytes (data); - g_autoptr(GMemoryOutputStream) memout = (GMemoryOutputStream*)g_memory_output_stream_new (NULL, 0, g_realloc, g_free); - g_autoptr(GInputStream) convin = g_converter_input_stream_new ((GInputStream*)memin, converter); - - { - gssize n_bytes_written = g_output_stream_splice ((GOutputStream*)memout, convin, - G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | - G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET, - cancellable, error); - if (n_bytes_written < 0) - goto out; - } - - ret = TRUE; - *out_uncompressed = g_memory_output_stream_steal_as_bytes (memout); - out: - return ret; -} - -gboolean -_ostree_static_delta_part_execute (OstreeRepo *repo, - GVariant *header, - GBytes *part_bytes, - gboolean trusted, - GCancellable *cancellable, - GError **error) -{ - gboolean ret = FALSE; - gsize partlen; - const guint8*partdata; - g_autoptr(GBytes) part_payload_bytes = NULL; - g_autoptr(GBytes) payload_data = NULL; - g_autoptr(GVariant) payload = NULL; - guint8 comptype; - - partdata = g_bytes_get_data (part_bytes, &partlen); - - if (partlen < 1) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Corrupted 0 length delta part"); - goto out; - } - - /* First byte is compression type */ - comptype = partdata[0]; - /* Then the rest may be compressed or uncompressed */ - part_payload_bytes = g_bytes_new_from_bytes (part_bytes, 1, partlen - 1); - switch (comptype) - { - case 0: - /* No compression */ - payload_data = g_bytes_ref (part_payload_bytes); - break; - case 'x': - { - g_autoptr(GConverter) decomp = - (GConverter*) _ostree_lzma_decompressor_new (); - - if (!decompress_all (decomp, part_payload_bytes, &payload_data, - cancellable, error)) - goto out; - } - break; - default: - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Invalid compression type '%u'", comptype); - goto out; - } - - payload = g_variant_new_from_bytes (G_VARIANT_TYPE (OSTREE_STATIC_DELTA_PART_PAYLOAD_FORMAT_V0), - payload_data, FALSE); - if (!_ostree_static_delta_part_execute_raw (repo, header, payload, trusted, - cancellable, error)) - goto out; - - ret = TRUE; - out: - return ret; -} - typedef struct { OstreeRepo *repo; GVariant *header; - GBytes *partdata; + GVariant *part; GCancellable *cancellable; GSimpleAsyncResult *result; gboolean trusted; @@ -385,7 +296,7 @@ static_delta_part_execute_async_data_free (gpointer user_data) g_clear_object (&data->repo); g_variant_unref (data->header); - g_bytes_unref (data->partdata); + g_variant_unref (data->part); g_clear_object (&data->cancellable); g_free (data); } @@ -401,8 +312,9 @@ static_delta_part_execute_thread (GSimpleAsyncResult *res, data = g_simple_async_result_get_op_res_gpointer (res); if (!_ostree_static_delta_part_execute (data->repo, data->header, - data->partdata, + data->part, data->trusted, + FALSE, NULL, cancellable, &error)) g_simple_async_result_take_error (res, error); } @@ -410,7 +322,7 @@ static_delta_part_execute_thread (GSimpleAsyncResult *res, void _ostree_static_delta_part_execute_async (OstreeRepo *repo, GVariant *header, - GBytes *partdata, + GVariant *part, gboolean trusted, GCancellable *cancellable, GAsyncReadyCallback callback, @@ -421,7 +333,7 @@ _ostree_static_delta_part_execute_async (OstreeRepo *repo, asyncdata = g_new0 (StaticDeltaPartExecuteAsyncData, 1); asyncdata->repo = g_object_ref (repo); asyncdata->header = g_variant_ref (header); - asyncdata->partdata = g_bytes_ref (partdata); + asyncdata->part = g_variant_ref (part); asyncdata->trusted = trusted; asyncdata->cancellable = cancellable ? g_object_ref (cancellable) : NULL; @@ -538,6 +450,12 @@ dispatch_bspatch (OstreeRepo *repo, if (!read_varuint64 (state, &length, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (!state->have_obj) { input_mfile = g_mapped_file_new_from_fd (state->read_source_fd, FALSE, error); @@ -596,6 +514,12 @@ dispatch_open_splice_and_close (OstreeRepo *repo, goto out; if (!validate_ofs (state, offset, length, error)) goto out; + + if (state->stats_only) + { + ret = TRUE; + goto out; + } metadata = g_variant_new_from_data (ostree_metadata_variant_type (state->output_objtype), state->payload_data + offset, length, TRUE, NULL, NULL); @@ -639,6 +563,12 @@ dispatch_open_splice_and_close (OstreeRepo *repo, if (!validate_ofs (state, content_offset, state->content_size, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + /* Fast path for regular files to bare repositories */ if (S_ISREG (state->mode) && (repo->mode == OSTREE_REPO_MODE_BARE || @@ -730,6 +660,8 @@ dispatch_open_splice_and_close (OstreeRepo *repo, ret = TRUE; out: + if (state->stats_only) + (void) dispatch_close (repo, state, cancellable, NULL); if (!ret) g_prefix_error (error, "opcode open-splice-and-close: "); return ret; @@ -745,8 +677,11 @@ dispatch_open (OstreeRepo *repo, g_assert (state->output_target == NULL); /* FIXME - lift this restriction */ - g_assert (repo->mode == OSTREE_REPO_MODE_BARE || - repo->mode == OSTREE_REPO_MODE_BARE_USER); + if (!state->stats_only) + { + g_assert (repo->mode == OSTREE_REPO_MODE_BARE || + repo->mode == OSTREE_REPO_MODE_BARE_USER); + } if (!open_output_target (state, cancellable, error)) goto out; @@ -757,6 +692,12 @@ dispatch_open (OstreeRepo *repo, if (!read_varuint64 (state, &state->content_size, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (state->trusted) { if (!_ostree_repo_open_trusted_content_bare (repo, state->checksum, @@ -801,6 +742,12 @@ dispatch_write (OstreeRepo *repo, if (!read_varuint64 (state, &content_offset, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (!state->have_obj) { if (state->read_source_fd != -1) @@ -881,6 +828,12 @@ dispatch_set_read_source (OstreeRepo *repo, if (!validate_ofs (state, source_offset, 32, error)) goto out; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + g_free (state->read_source_object); state->read_source_object = ostree_checksum_from_bytes (state->payload_data + source_offset); @@ -903,6 +856,12 @@ dispatch_unset_read_source (OstreeRepo *repo, { gboolean ret = FALSE; + if (state->stats_only) + { + ret = TRUE; + goto out; + } + if (state->read_source_fd) { (void) close (state->read_source_fd); @@ -912,7 +871,7 @@ dispatch_unset_read_source (OstreeRepo *repo, g_clear_pointer (&state->read_source_object, g_free); ret = TRUE; - /* out: */ + out: if (!ret) g_prefix_error (error, "opcode unset-read-source: "); return ret; diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 415421af..3b08d445 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -518,8 +518,6 @@ ostree_repo_finalize (GObject *object) g_clear_object (&self->tmp_dir); if (self->tmp_dir_fd) (void) close (self->tmp_dir_fd); - g_clear_object (&self->local_heads_dir); - g_clear_object (&self->remote_heads_dir); g_clear_object (&self->objects_dir); if (self->objects_dir_fd != -1) (void) close (self->objects_dir_fd); @@ -605,8 +603,6 @@ ostree_repo_constructed (GObject *object) g_assert (self->repodir != NULL); self->tmp_dir = g_file_resolve_relative_path (self->repodir, "tmp"); - self->local_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/heads"); - self->remote_heads_dir = g_file_resolve_relative_path (self->repodir, "refs/remotes"); self->objects_dir = g_file_get_child (self->repodir, "objects"); self->deltas_dir = g_file_get_child (self->repodir, "deltas"); @@ -2384,6 +2380,19 @@ ostree_repo_set_disable_fsync (OstreeRepo *self, self->disable_fsync = disable_fsync; } +/** + * ostree_repo_get_disable_fsync: + * @self: An #OstreeRepo + * + * For more information see ostree_repo_set_disable_fsync(). + * + * Returns: Whether or not fsync() is enabled for this repo. + */ +gboolean +ostree_repo_get_disable_fsync (OstreeRepo *self) +{ + return self->disable_fsync; +} /* Replace the contents of a file, honoring the repository's fsync * policy. @@ -3775,6 +3784,7 @@ ostree_repo_pull_one_dir (OstreeRepo *self, * * flags (i): An instance of #OstreeRepoPullFlags * * refs: (as): Array of string refs * * depth: (i): How far in the history to traverse; default is 0, -1 means infinite + * * override-commit-ids: (as): Array of specific commit IDs to fetch for refs */ gboolean ostree_repo_pull_with_options (OstreeRepo *self, @@ -4594,8 +4604,8 @@ ostree_repo_regenerate_summary (OstreeRepo *self, gs_free char *from = NULL; gs_free char *to = NULL; gs_free guchar *csum = NULL; - gs_free char *superblock; - gs_fd_close int superblock_file_fd; + gs_free char *superblock = NULL; + gs_fd_close int superblock_file_fd = -1; g_autoptr(GInputStream) in_stream = NULL; _ostree_parse_delta_name (delta_names->pdata[i], &from, &to); diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index 5bc25204..4629ea55 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -56,6 +56,8 @@ gboolean ostree_repo_open (OstreeRepo *self, void ostree_repo_set_disable_fsync (OstreeRepo *self, gboolean disable_fsync); +gboolean ostree_repo_get_disable_fsync (OstreeRepo *self); + gboolean ostree_repo_is_system (OstreeRepo *repo); gboolean ostree_repo_is_writable (OstreeRepo *self, @@ -439,6 +441,7 @@ gboolean ostree_repo_write_dfd_to_mtree (OstreeRepo *self, GCancellable *cancellable, GError **error); + gboolean ostree_repo_write_archive_to_mtree (OstreeRepo *self, GFile *archive, OstreeMutableTree *mtree, @@ -447,6 +450,53 @@ gboolean ostree_repo_write_archive_to_mtree (OstreeRepo * GCancellable *cancellable, GError **error); +/** + * OstreeRepoImportArchiveOptions: + * + * An extensible options structure controlling archive import. Ensure that + * you have entirely zeroed the structure, then set just the desired + * options. This is used by ostree_repo_import_archive_to_mtree(). + */ +typedef struct { + guint ignore_unsupported_content : 1; + guint autocreate_parents : 1; + guint reserved : 30; + + guint unused_uint[8]; + gpointer unused_ptrs[8]; +} OstreeRepoImportArchiveOptions; + +gboolean ostree_repo_import_archive_to_mtree (OstreeRepo *self, + OstreeRepoImportArchiveOptions *opts, + void *archive, /* Really struct archive * */ + OstreeMutableTree *mtree, + OstreeRepoCommitModifier *modifier, + GCancellable *cancellable, + GError **error); +/** + * OstreeRepoExportArchiveOptions: + * + * An extensible options structure controlling archive creation. Ensure that + * you have entirely zeroed the structure, then set just the desired + * options. This is used by ostree_repo_export_tree_to_archive(). + */ +typedef struct { + guint disable_xattrs : 1; + guint reserved : 31; + + guint64 timestamp_secs; + + guint unused_uint[8]; + gpointer unused_ptrs[8]; +} OstreeRepoExportArchiveOptions; + +gboolean ostree_repo_export_tree_to_archive (OstreeRepo *self, + OstreeRepoExportArchiveOptions *opts, + OstreeRepoFile *root, + void *archive, /* Really struct archive * */ + GCancellable *cancellable, + GError **error); + gboolean ostree_repo_write_mtree (OstreeRepo *self, OstreeMutableTree *mtree, GFile **out_file, @@ -530,7 +580,9 @@ typedef struct { OstreeRepoCheckoutOverwriteMode overwrite_mode; guint enable_uncompressed_cache : 1; - guint unused : 31; + guint disable_fsync : 1; + guint process_whiteouts : 1; + guint reserved : 29; const char *subpath; diff --git a/src/libostree/ostree-sysroot-deploy.c b/src/libostree/ostree-sysroot-deploy.c index 041b0b25..aa034951 100644 --- a/src/libostree/ostree-sysroot-deploy.c +++ b/src/libostree/ostree-sysroot-deploy.c @@ -528,6 +528,11 @@ checkout_deployment_tree (OstreeSysroot *sysroot, glnx_fd_close int osdeploy_dfd = -1; int ret_fd; + /* We end up using syncfs for the entire filesystem, so turn off + * OstreeRepo level fsync. + */ + checkout_opts.disable_fsync = TRUE; + osdeploy_path = g_strconcat ("ostree/deploy/", ostree_deployment_get_osname (deployment), "/deploy", NULL); checkout_target_name = g_strdup_printf ("%s.%d", csum, ostree_deployment_get_deployserial (deployment)); @@ -2030,9 +2035,12 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self, cancellable, error)) goto out; - if (!ostree_sysroot_deployment_set_mutable (self, new_deployment, FALSE, - cancellable, error)) - goto out; + if (!(self->debug_flags & OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS)) + { + if (!ostree_sysroot_deployment_set_mutable (self, new_deployment, FALSE, + cancellable, error)) + goto out; + } { ostree_cleanup_sepolicy_fscreatecon gpointer dummy = NULL; diff --git a/src/libostree/ostree-sysroot-private.h b/src/libostree/ostree-sysroot-private.h index 0c38e269..e0dc24fd 100644 --- a/src/libostree/ostree-sysroot-private.h +++ b/src/libostree/ostree-sysroot-private.h @@ -27,6 +27,13 @@ G_BEGIN_DECLS +typedef enum { + + /* Don't flag deployments as immutable. */ + OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS = 1 << 0 + +} OstreeSysrootDebugFlags; + struct OstreeSysroot { GObject parent; @@ -46,6 +53,7 @@ struct OstreeSysroot { /* Only access through ostree_sysroot_get_repo() */ OstreeRepo *repo; + OstreeSysrootDebugFlags debug_flags; }; #define OSTREE_SYSROOT_LOCKFILE "ostree/lock" diff --git a/src/libostree/ostree-sysroot.c b/src/libostree/ostree-sysroot.c index 7a4686d8..5ad2713a 100644 --- a/src/libostree/ostree-sysroot.c +++ b/src/libostree/ostree-sysroot.c @@ -153,6 +153,13 @@ ostree_sysroot_class_init (OstreeSysrootClass *klass) static void ostree_sysroot_init (OstreeSysroot *self) { + const GDebugKey keys[] = { + { "mutable-deployments", OSTREE_SYSROOT_DEBUG_MUTABLE_DEPLOYMENTS }, + }; + + self->debug_flags = g_parse_debug_string (g_getenv("OSTREE_SYSROOT_DEBUG"), + keys, G_N_ELEMENTS (keys)); + self->sysroot_fd = -1; self->lock = (GLnxLockFile)GLNX_LOCK_FILE_INIT; } diff --git a/src/libotutil/ot-gpg-utils.c b/src/libotutil/ot-gpg-utils.c index aa5b4819..9414088d 100644 --- a/src/libotutil/ot-gpg-utils.c +++ b/src/libotutil/ot-gpg-utils.c @@ -31,6 +31,7 @@ ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, GError **error) { GIOErrorEnum errcode; + char errbuf[1024]; /* XXX This list is incomplete. Add cases as needed. */ @@ -42,9 +43,11 @@ ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, /* special case - abort on out-of-memory */ case GPG_ERR_ENOMEM: + (void) gpg_strerror_r (gpg_error, errbuf, sizeof (errbuf)); + errbuf[sizeof(errbuf)-1] = '\0'; g_error ("%s: %s", gpgme_strsource (gpg_error), - gpgme_strerror (gpg_error)); + errbuf); case GPG_ERR_INV_VALUE: errcode = G_IO_ERROR_INVALID_ARGUMENT; @@ -55,9 +58,11 @@ ot_gpgme_error_to_gio_error (gpgme_error_t gpg_error, break; } + (void) gpg_strerror_r (gpg_error, errbuf, sizeof (errbuf)); + errbuf[sizeof(errbuf)-1] = '\0'; g_set_error (error, G_IO_ERROR, errcode, "%s: %s", gpgme_strsource (gpg_error), - gpgme_strerror (gpg_error)); + errbuf); } gboolean diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index 1210a785..ed650268 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -154,6 +154,28 @@ ot_util_variant_map (GFile *src, return ret; } +gboolean +ot_util_variant_map_at (int dfd, + const char *path, + const GVariantType *type, + gboolean trusted, + GVariant **out_variant, + GError **error) +{ + glnx_fd_close int fd = -1; + g_autoptr(GVariant) ret_variant = NULL; + + fd = openat (dfd, path, O_RDONLY | O_CLOEXEC); + if (fd < 0) + { + glnx_set_error_from_errno (error); + g_prefix_error (error, "Opening %s: ", path); + return FALSE; + } + + return ot_util_variant_map_fd (fd, 0, type, trusted, out_variant, error); +} + typedef struct { gpointer addr; gsize len; diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h index 34b6eb5d..1a7abe0e 100644 --- a/src/libotutil/ot-variant-utils.h +++ b/src/libotutil/ot-variant-utils.h @@ -48,6 +48,13 @@ gboolean ot_util_variant_map (GFile *src, GVariant **out_variant, GError **error); +gboolean ot_util_variant_map_at (int dfd, + const char *path, + const GVariantType *type, + gboolean trusted, + GVariant **out_variant, + GError **error); + gboolean ot_util_variant_map_fd (int fd, goffset offset, const GVariantType *type, diff --git a/src/ostree/main.c b/src/ostree/main.c index 99d7d916..eff3082d 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -40,6 +40,7 @@ static OstreeCommand commands[] = { { "commit", ostree_builtin_commit }, { "config", ostree_builtin_config }, { "diff", ostree_builtin_diff }, + { "export", ostree_builtin_export }, { "fsck", ostree_builtin_fsck }, { "gpg-sign", ostree_builtin_gpg_sign }, { "init", ostree_builtin_init }, diff --git a/src/ostree/ot-builtin-checkout.c b/src/ostree/ot-builtin-checkout.c index 9929a37b..810a8f72 100644 --- a/src/ostree/ot-builtin-checkout.c +++ b/src/ostree/ot-builtin-checkout.c @@ -36,6 +36,7 @@ static gboolean opt_allow_noent; static gboolean opt_disable_cache; static char *opt_subpath; static gboolean opt_union; +static gboolean opt_whiteouts; static gboolean opt_from_stdin; static char *opt_from_file; static gboolean opt_disable_fsync; @@ -61,6 +62,7 @@ static GOptionEntry options[] = { { "disable-cache", 0, 0, G_OPTION_ARG_NONE, &opt_disable_cache, "Do not update or use the internal repository uncompressed object cache", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Checkout sub-directory PATH", "PATH" }, { "union", 0, 0, G_OPTION_ARG_NONE, &opt_union, "Keep existing directories, overwrite existing files", NULL }, + { "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL }, { "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL }, { "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL }, { "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" }, @@ -83,7 +85,7 @@ process_one_checkout (OstreeRepo *repo, * `ostree_repo_checkout_tree_at` until such time as we have a more * convenient infrastructure for testing C APIs with data. */ - if (opt_disable_cache) + if (opt_disable_cache || opt_whiteouts) { OstreeRepoCheckoutOptions options = { 0, }; @@ -91,10 +93,11 @@ process_one_checkout (OstreeRepo *repo, options.mode = OSTREE_REPO_CHECKOUT_MODE_USER; if (opt_union) options.overwrite_mode = OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES; + if (opt_whiteouts) + options.process_whiteouts = TRUE; if (subpath) options.subpath = subpath; - if (!ostree_repo_checkout_tree_at (repo, &options, AT_FDCWD, destination, resolved_commit, diff --git a/src/ostree/ot-builtin-diff.c b/src/ostree/ot-builtin-diff.c index 2a27f5d2..a23ed83f 100644 --- a/src/ostree/ot-builtin-diff.c +++ b/src/ostree/ot-builtin-diff.c @@ -29,10 +29,12 @@ static gboolean opt_stats; static gboolean opt_fs_diff; +static gboolean opt_no_xattrs; static GOptionEntry options[] = { { "stats", 0, 0, G_OPTION_ARG_NONE, &opt_stats, "Print various statistics", NULL }, { "fs-diff", 0, 0, G_OPTION_ARG_NONE, &opt_fs_diff, "Print filesystem diff", NULL }, + { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, { NULL } }; @@ -162,6 +164,11 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_fs_diff) { + OstreeDiffFlags diff_flags = OSTREE_DIFF_FLAGS_NONE; + + if (opt_no_xattrs) + diff_flags |= OSTREE_DIFF_FLAGS_IGNORE_XATTRS; + if (!parse_file_or_commit (repo, src, &srcf, cancellable, error)) goto out; if (!parse_file_or_commit (repo, target, &targetf, cancellable, error)) @@ -171,7 +178,7 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** removed = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); added = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - if (!ostree_diff_dirs (OSTREE_DIFF_FLAGS_NONE, srcf, targetf, modified, removed, added, cancellable, error)) + if (!ostree_diff_dirs (diff_flags, srcf, targetf, modified, removed, added, cancellable, error)) goto out; ostree_diff_print (srcf, targetf, modified, removed, added); @@ -194,9 +201,9 @@ ostree_builtin_diff (int argc, char **argv, GCancellable *cancellable, GError ** if (!ostree_repo_resolve_rev (repo, target, FALSE, &rev_b, error)) goto out; - if (!ostree_repo_traverse_commit (repo, rev_a, -1, &reachable_a, cancellable, error)) + if (!ostree_repo_traverse_commit (repo, rev_a, 0, &reachable_a, cancellable, error)) goto out; - if (!ostree_repo_traverse_commit (repo, rev_b, -1, &reachable_b, cancellable, error)) + if (!ostree_repo_traverse_commit (repo, rev_b, 0, &reachable_b, cancellable, error)) goto out; a_size = g_hash_table_size (reachable_a); diff --git a/src/ostree/ot-builtin-export.c b/src/ostree/ot-builtin-export.c new file mode 100644 index 00000000..cccb50e7 --- /dev/null +++ b/src/ostree/ot-builtin-export.c @@ -0,0 +1,148 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ot-main.h" +#include "ot-builtins.h" +#include "ostree.h" +#include "ostree-repo-file.h" +#include "otutil.h" + +#ifdef HAVE_LIBARCHIVE +#include +#include +#endif + +static char *opt_output_path; +static gboolean opt_no_xattrs; + +static GOptionEntry options[] = { + { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Skip output of extended attributes", NULL }, + { "output", 'o', 0, G_OPTION_ARG_STRING, &opt_output_path, "Output to PATH ", "PATH" }, + { NULL } +}; + +#ifdef HAVE_LIBARCHIVE + +static void +propagate_libarchive_error (GError **error, + struct archive *a) +{ + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "%s", archive_error_string (a)); +} + +#endif + +gboolean +ostree_builtin_export (int argc, char **argv, GCancellable *cancellable, GError **error) +{ + GOptionContext *context; + glnx_unref_object OstreeRepo *repo = NULL; + gboolean ret = FALSE; + const char *rev; + g_autoptr(GFile) root = NULL; + g_autofree char *commit = NULL; + g_autoptr(GVariant) commit_data = NULL; + struct archive *a; + OstreeRepoExportArchiveOptions opts = { 0, }; + + context = g_option_context_new ("COMMIT - Stream COMMIT to stdout in tar format"); + + if (!ostree_option_context_parse (context, options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) + goto out; + +#ifdef HAVE_LIBARCHIVE + + if (argc <= 1) + { + ot_util_usage_error (context, "A COMMIT argument is required", error); + goto out; + } + rev = argv[1]; + + a = archive_write_new (); + /* Yes, this is hardcoded for now. There is + * archive_write_set_format_filter_by_ext() but it's fairly magic. + * Many programs have support now for GNU tar, so should be a good + * default. I also don't want to lock us into everything libarchive + * supports. + */ + if (archive_write_set_format_gnutar (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + if (archive_write_add_filter_none (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + if (opt_output_path) + { + if (archive_write_open_filename (a, opt_output_path) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + } + else + { + if (archive_write_open_FILE (a, stdout) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + } + + if (opt_no_xattrs) + opts.disable_xattrs = TRUE; + + if (!ostree_repo_read_commit (repo, rev, &root, &commit, cancellable, error)) + goto out; + + if (!ostree_repo_load_variant (repo, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_data, error)) + goto out; + + opts.timestamp_secs = ostree_commit_get_timestamp (commit_data); + + if (!ostree_repo_export_tree_to_archive (repo, &opts, (OstreeRepoFile*)root, a, + cancellable, error)) + goto out; + + if (archive_write_close (a) != ARCHIVE_OK) + { + propagate_libarchive_error (error, a); + goto out; + } + +#else + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED, + "This version of ostree is not compiled with libarchive support"); + goto out; +#endif + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + return ret; +} diff --git a/src/ostree/ot-builtin-pull.c b/src/ostree/ot-builtin-pull.c index 6aae8205..7c91890f 100644 --- a/src/ostree/ot-builtin-pull.c +++ b/src/ostree/ot-builtin-pull.c @@ -30,16 +30,20 @@ static gboolean opt_disable_fsync; static gboolean opt_mirror; static gboolean opt_commit_only; +static gboolean opt_dry_run; static gboolean opt_disable_static_deltas; +static gboolean opt_require_static_deltas; static char* opt_subpath; static int opt_depth = 0; - static GOptionEntry options[] = { +static GOptionEntry options[] = { { "commit-metadata-only", 0, 0, G_OPTION_ARG_NONE, &opt_commit_only, "Fetch only the commit metadata", NULL }, { "disable-fsync", 0, 0, G_OPTION_ARG_NONE, &opt_disable_fsync, "Do not invoke fsync()", NULL }, { "disable-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_disable_static_deltas, "Do not use static deltas", NULL }, + { "require-static-deltas", 0, 0, G_OPTION_ARG_NONE, &opt_require_static_deltas, "Require static deltas", NULL }, { "mirror", 0, 0, G_OPTION_ARG_NONE, &opt_mirror, "Write refs suitable for a mirror", NULL }, { "subpath", 0, 0, G_OPTION_ARG_STRING, &opt_subpath, "Only pull the provided subpath", NULL }, + { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Only print information on what will be downloaded (requires static deltas)", NULL }, { "depth", 0, 0, G_OPTION_ARG_INT, &opt_depth, "Traverse DEPTH parents (-1=infinite) (default: 0)", "DEPTH" }, { NULL } }; @@ -60,6 +64,39 @@ gpg_verify_result_cb (OstreeRepo *repo, gs_console_begin_status_line (console, "", NULL, NULL); } +static gboolean printed_console_progress; + +static void +dry_run_console_progress_changed (OstreeAsyncProgress *progress, + gpointer user_data) +{ + guint fetched_delta_parts, total_delta_parts; + guint64 total_delta_part_size, total_delta_part_usize; + GString *buf; + + g_assert (!printed_console_progress); + printed_console_progress = TRUE; + + fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts"); + total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts"); + total_delta_part_size = ostree_async_progress_get_uint64 (progress, "total-delta-part-size"); + total_delta_part_usize = ostree_async_progress_get_uint64 (progress, "total-delta-part-usize"); + + buf = g_string_new (""); + + { g_autofree char *formatted_size = + g_format_size (total_delta_part_size); + g_autofree char *formatted_usize = + g_format_size (total_delta_part_usize); + + g_string_append_printf (buf, "Delta update: %u/%u parts, %s to transfer, %s uncompressed", + fetched_delta_parts, total_delta_parts, + formatted_size, formatted_usize); + } + g_print ("%s\n", buf->str); + g_string_free (buf, TRUE); +} + gboolean ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError **error) { @@ -70,6 +107,7 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** OstreeRepoPullFlags pullflags = 0; GSConsole *console = NULL; g_autoptr(GPtrArray) refs_to_fetch = NULL; + g_autoptr(GPtrArray) override_commit_ids = NULL; glnx_unref_object OstreeAsyncProgress *progress = NULL; gulong signal_handler_id = 0; @@ -96,15 +134,49 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (opt_commit_only) pullflags |= OSTREE_REPO_PULL_FLAGS_COMMIT_ONLY; + if (opt_dry_run && !opt_require_static_deltas) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "--dry-run requires --require-static-deltas"); + goto out; + } + if (strchr (argv[1], ':') == NULL) { remote = g_strdup (argv[1]); if (argc > 2) { int i; - refs_to_fetch = g_ptr_array_new (); + refs_to_fetch = g_ptr_array_new_with_free_func (g_free); + for (i = 2; i < argc; i++) - g_ptr_array_add (refs_to_fetch, argv[i]); + { + const char *at = strrchr (argv[i], '@'); + + if (at) + { + guint j; + const char *override_commit_id = at + 1; + + if (!ostree_validate_checksum_string (override_commit_id, error)) + goto out; + + if (!override_commit_ids) + override_commit_ids = g_ptr_array_new_with_free_func (g_free); + + /* Backfill */ + for (j = 2; j < i; i++) + g_ptr_array_add (override_commit_ids, g_strdup ("")); + + g_ptr_array_add (override_commit_ids, g_strdup (override_commit_id)); + g_ptr_array_add (refs_to_fetch, g_strndup (argv[i], at - argv[i])); + } + else + { + g_ptr_array_add (refs_to_fetch, g_strdup (argv[i])); + } + } + g_ptr_array_add (refs_to_fetch, NULL); } } @@ -119,11 +191,21 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_ptr_array_add (refs_to_fetch, NULL); } - console = gs_console_get (); - if (console) + if (!opt_dry_run) { - gs_console_begin_status_line (console, "", NULL, NULL); - progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); + console = gs_console_get (); + if (console) + { + gs_console_begin_status_line (console, "", NULL, NULL); + progress = ostree_async_progress_new_and_connect (ostree_repo_pull_default_console_progress_changed, console); + signal_handler_id = g_signal_connect (repo, "gpg-verify-result", + G_CALLBACK (gpg_verify_result_cb), + console); + } + } + else + { + progress = ostree_async_progress_new_and_connect (dry_run_console_progress_changed, console); signal_handler_id = g_signal_connect (repo, "gpg-verify-result", G_CALLBACK (gpg_verify_result_cb), console); @@ -147,6 +229,16 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** g_variant_builder_add (&builder, "{s@v}", "disable-static-deltas", g_variant_new_variant (g_variant_new_boolean (opt_disable_static_deltas))); + g_variant_builder_add (&builder, "{s@v}", "require-static-deltas", + g_variant_new_variant (g_variant_new_boolean (opt_require_static_deltas))); + + g_variant_builder_add (&builder, "{s@v}", "dry-run", + g_variant_new_variant (g_variant_new_boolean (opt_dry_run))); + + if (override_commit_ids) + g_variant_builder_add (&builder, "{s@v}", "override-commit-ids", + g_variant_new_variant (g_variant_new_strv ((const char*const*)override_commit_ids->pdata, override_commit_ids->len))); + if (!ostree_repo_pull_with_options (repo, remote, g_variant_builder_end (&builder), progress, cancellable, error)) goto out; @@ -155,6 +247,9 @@ ostree_builtin_pull (int argc, char **argv, GCancellable *cancellable, GError ** if (progress) ostree_async_progress_finish (progress); + if (opt_dry_run) + g_assert (printed_console_progress); + ret = TRUE; out: if (signal_handler_id > 0) diff --git a/src/ostree/ot-builtin-static-delta.c b/src/ostree/ot-builtin-static-delta.c index 903e5fd6..36fb63fa 100644 --- a/src/ostree/ot-builtin-static-delta.c +++ b/src/ostree/ot-builtin-static-delta.c @@ -23,6 +23,7 @@ #include "ot-main.h" #include "ot-builtins.h" #include "ostree.h" +#include "ostree-cmdprivate.h" #include "ot-main.h" #include "otutil.h" @@ -31,13 +32,16 @@ static char *opt_to_rev; static char *opt_min_fallback_size; static char *opt_max_bsdiff_size; static char *opt_max_chunk_size; +static char *opt_endianness; static gboolean opt_empty; +static gboolean opt_swap_endianness; static gboolean opt_inline; static gboolean opt_disable_bsdiff; #define BUILTINPROTO(name) static gboolean ot_static_delta_builtin_ ## name (int argc, char **argv, GCancellable *cancellable, GError **error) BUILTINPROTO(list); +BUILTINPROTO(show); BUILTINPROTO(generate); BUILTINPROTO(apply_offline); @@ -45,6 +49,7 @@ BUILTINPROTO(apply_offline); static OstreeCommand static_delta_subcommands[] = { { "list", ot_static_delta_builtin_list }, + { "show", ot_static_delta_builtin_show }, { "generate", ot_static_delta_builtin_generate }, { "apply-offline", ot_static_delta_builtin_apply_offline }, { NULL, NULL } @@ -57,6 +62,8 @@ static GOptionEntry generate_options[] = { { "inline", 0, 0, G_OPTION_ARG_NONE, &opt_inline, "Inline delta parts into main delta", NULL }, { "to", 0, 0, G_OPTION_ARG_STRING, &opt_to_rev, "Create delta to revision REV", "REV" }, { "disable-bsdiff", 0, 0, G_OPTION_ARG_NONE, &opt_disable_bsdiff, "Disable use of bsdiff", NULL }, + { "set-endianness", 0, 0, G_OPTION_ARG_STRING, &opt_endianness, "Choose metadata endianness ('l' or 'B')", "ENDIAN" }, + { "swap-endianness", 0, 0, G_OPTION_ARG_NONE, &opt_swap_endianness, "Swap metadata endianness from host order", NULL }, { "min-fallback-size", 0, 0, G_OPTION_ARG_STRING, &opt_min_fallback_size, "Minimum uncompressed size in megabytes for individual HTTP request", NULL}, { "max-bsdiff-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_bsdiff_size, "Maximum size in megabytes to consider bsdiff compression for input files", NULL}, { "max-chunk-size", 0, 0, G_OPTION_ARG_STRING, &opt_max_chunk_size, "Maximum size of delta chunks in megabytes", NULL}, @@ -129,6 +136,38 @@ ot_static_delta_builtin_list (int argc, char **argv, GCancellable *cancellable, return ret; } +static gboolean +ot_static_delta_builtin_show (int argc, char **argv, GCancellable *cancellable, GError **error) +{ + gboolean ret = FALSE; + GOptionContext *context; + glnx_unref_object OstreeRepo *repo = NULL; + const char *delta_id = NULL; + + context = g_option_context_new ("SHOW - Dump information on a delta"); + + if (!ostree_option_context_parse (context, list_options, &argc, &argv, OSTREE_BUILTIN_FLAG_NONE, &repo, cancellable, error)) + goto out; + + if (argc < 3) + { + g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "DELTA must be specified"); + goto out; + } + + delta_id = argv[2]; + + if (!ostree_cmd__private__ ()->ostree_static_delta_dump (repo, delta_id, cancellable, error)) + goto out; + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + return ret; +} + static gboolean ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellable, GError **error) { @@ -159,6 +198,7 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab g_autofree char *to_resolved = NULL; g_autofree char *from_parent_str = NULL; g_autoptr(GVariantBuilder) parambuilder = NULL; + int endianness; g_assert (opt_to_rev); @@ -189,6 +229,37 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab } if (!ostree_repo_resolve_rev (repo, opt_to_rev, FALSE, &to_resolved, error)) goto out; + + if (opt_endianness) + { + if (strcmp (opt_endianness, "l") == 0) + endianness = G_LITTLE_ENDIAN; + else if (strcmp (opt_endianness, "B") == 0) + endianness = G_BIG_ENDIAN; + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid endianness '%s'", opt_endianness); + goto out; + } + } + else + endianness = G_BYTE_ORDER; + + if (opt_swap_endianness) + { + switch (endianness) + { + case G_LITTLE_ENDIAN: + endianness = G_BIG_ENDIAN; + break; + case G_BIG_ENDIAN: + endianness = G_LITTLE_ENDIAN; + break; + default: + g_assert_not_reached (); + } + } parambuilder = g_variant_builder_new (G_VARIANT_TYPE ("a{sv}")); if (opt_min_fallback_size) @@ -208,6 +279,8 @@ ot_static_delta_builtin_generate (int argc, char **argv, GCancellable *cancellab "inline-parts", g_variant_new_boolean (TRUE)); g_variant_builder_add (parambuilder, "{sv}", "verbose", g_variant_new_boolean (TRUE)); + if (opt_endianness || opt_swap_endianness) + g_variant_builder_add (parambuilder, "{sv}", "endianness", g_variant_new_uint32 (endianness)); g_print ("Generating static delta:\n"); g_print (" From: %s\n", from_resolved ? from_resolved : "empty"); diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index 95262ec4..1c862925 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -35,6 +35,7 @@ BUILTINPROTO(checkout); BUILTINPROTO(checksum); BUILTINPROTO(commit); BUILTINPROTO(diff); +BUILTINPROTO(export); BUILTINPROTO(gpg_sign); BUILTINPROTO(init); BUILTINPROTO(log); diff --git a/src/rofiles-fuse/Makefile-inc.am b/src/rofiles-fuse/Makefile-inc.am new file mode 100644 index 00000000..5510a2bd --- /dev/null +++ b/src/rofiles-fuse/Makefile-inc.am @@ -0,0 +1,23 @@ +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +bin_PROGRAMS += rofiles-fuse + +rofiles_fuse_SOURCES = src/rofiles-fuse/main.c + +rofiles_fuse_CFLAGS = $(AM_CFLAGS) -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 $(BUILDOPT_FUSE_CFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -I$(srcdir)/libglnx $(NULL) +rofiles_fuse_LDADD = libglnx.la $(BUILDOPT_FUSE_LIBS) $(OT_INTERNAL_GIO_UNIX_LIBS) diff --git a/src/rofiles-fuse/README.md b/src/rofiles-fuse/README.md new file mode 100644 index 00000000..1f18afcc --- /dev/null +++ b/src/rofiles-fuse/README.md @@ -0,0 +1,48 @@ +rofiles-fuse +============ + +Create a mountpoint that represents an underlying directory hierarchy, +but where non-directory inodes cannot have content or xattrs changed. +Files can still be unlinked, and new ones created. + +This filesystem is designed for OSTree and other systems that create +"hardlink farms", i.e. filesystem trees deduplicated via hardlinks. + +Normally with hard links, if you change one, you change them all. + +There are two approaches to dealing with that: + - Copy on write: implemented by BTRFS, overlayfs, and http://linux-vserver.org/util-vserver:Vhashify + - Make them read-only: what this FUSE mount does + +Usage +===== + +Let's say that you have immutable data in `/srv/backups/20150410`, and +you want to update it with a new version, storing the result in +`/srv/backups/20150411`. Further assume that all software operating +on the directory does the "create tempfile and `rename()`" dance +rather than in-place edits. + + $ mkdir -p /srv/backups/mnt # Ensure we have a mount point + $ cp -al /srv/backups/20150410 /srv/backups/20150411 + $ rofiles-fuse /srv/backups/20150411 /srv/backups/mnt + +Now we have a "rofiles" mount at `/srv/backups/mnt`. If we try this: + + $ echo new doc content > /srv/backups/mnt/document + bash: /srv/backups/mnt/document: Read-only file system + +It failed because the `>` redirection operator will try to truncate +the existing file. If instead we create `document.tmp` and then +rename it atomically over the old one, it will work: + + $ echo new doc content > /srv/backups/mnt/document.tmp + $ mv /srv/backups/mnt/document.tmp /srv/backups/mnt/document + +Let's unmount: + + $ fusermount -u /srv/backups/mnt + +Now we have two directories `/srv/backups/20150410` +`/srv/backups/20150411` which share all file storage except for the +new document. diff --git a/src/rofiles-fuse/main.c b/src/rofiles-fuse/main.c new file mode 100644 index 00000000..3c910f1f --- /dev/null +++ b/src/rofiles-fuse/main.c @@ -0,0 +1,601 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015,2016 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#define FUSE_USE_VERSION 26 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "libglnx.h" + +// Global to store our read-write path +static int basefd = -1; +static GHashTable *created_devino_hash = NULL; + +static inline const char * +ENSURE_RELPATH (const char *path) +{ + path = path + strspn (path, "/"); + if (*path == 0) + return "."; + return path; +} + +typedef struct { + dev_t dev; + ino_t ino; +} DevIno; + +static guint +devino_hash (gconstpointer a) +{ + DevIno *a_i = (gpointer)a; + return (guint) (a_i->dev + a_i->ino); +} + +static int +devino_equal (gconstpointer a, + gconstpointer b) +{ + DevIno *a_i = (gpointer)a; + DevIno *b_i = (gpointer)b; + return a_i->dev == b_i->dev + && a_i->ino == b_i->ino; +} + +static gboolean +devino_set_contains (dev_t dev, ino_t ino) +{ + DevIno devino = { dev, ino }; + return g_hash_table_contains (created_devino_hash, &devino); +} + +static gboolean +devino_set_insert (dev_t dev, ino_t ino) +{ + DevIno *devino = g_new (DevIno, 1); + devino->dev = dev; + devino->ino = ino; + return g_hash_table_add (created_devino_hash, devino); +} + +static gboolean +devino_set_remove (dev_t dev, ino_t ino) +{ + DevIno devino = { dev, ino }; + return g_hash_table_remove (created_devino_hash, &devino); +} + +static int +callback_getattr (const char *path, struct stat *st_data) +{ + path = ENSURE_RELPATH (path); + if (!*path) + { + if (fstat (basefd, st_data) == -1) + return -errno; + } + else + { + if (fstatat (basefd, path, st_data, AT_SYMLINK_NOFOLLOW) == -1) + return -errno; + } + return 0; +} + +static int +callback_readlink (const char *path, char *buf, size_t size) +{ + int r; + + path = ENSURE_RELPATH (path); + + /* Note FUSE wants the string to be always nul-terminated, even if + * truncated. + */ + r = readlinkat (basefd, path, buf, size - 1); + if (r == -1) + return -errno; + buf[r] = '\0'; + return 0; +} + +static int +callback_readdir (const char *path, void *buf, fuse_fill_dir_t filler, + off_t offset, struct fuse_file_info *fi) +{ + DIR *dp; + struct dirent *de; + int dfd; + + path = ENSURE_RELPATH (path); + + if (!*path) + { + dfd = fcntl (basefd, F_DUPFD_CLOEXEC, 3); + lseek (dfd, 0, SEEK_SET); + } + else + { + dfd = openat (basefd, path, O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY); + if (dfd == -1) + return -errno; + } + + /* Transfers ownership of fd */ + dp = fdopendir (dfd); + if (dp == NULL) + return -errno; + + while ((de = readdir (dp)) != NULL) + { + struct stat st; + memset (&st, 0, sizeof (st)); + st.st_ino = de->d_ino; + st.st_mode = de->d_type << 12; + if (filler (buf, de->d_name, &st, 0)) + break; + } + + (void) closedir (dp); + return 0; +} + +static int +callback_mknod (const char *path, mode_t mode, dev_t rdev) +{ + return -EROFS; +} + +static int +callback_mkdir (const char *path, mode_t mode) +{ + path = ENSURE_RELPATH (path); + if (mkdirat (basefd, path, mode) == -1) + return -errno; + return 0; +} + +static int +callback_unlink (const char *path) +{ + struct stat stbuf; + path = ENSURE_RELPATH (path); + + if (fstatat (basefd, path, &stbuf, AT_SYMLINK_NOFOLLOW) == 0) + { + if (!S_ISDIR (stbuf.st_mode)) + devino_set_remove (stbuf.st_dev, stbuf.st_ino); + } + + if (unlinkat (basefd, path, 0) == -1) + return -errno; + return 0; +} + +static int +callback_rmdir (const char *path) +{ + path = ENSURE_RELPATH (path); + if (unlinkat (basefd, path, AT_REMOVEDIR) == -1) + return -errno; + return 0; +} + +static int +callback_symlink (const char *from, const char *to) +{ + struct stat stbuf; + + to = ENSURE_RELPATH (to); + + if (symlinkat (from, basefd, to) == -1) + return -errno; + + if (fstatat (basefd, to, &stbuf, AT_SYMLINK_NOFOLLOW) == -1) + { + fprintf (stderr, "Failed to find newly created symlink '%s': %s\n", + to, g_strerror (errno)); + exit (1); + } + return 0; +} + +static int +callback_rename (const char *from, const char *to) +{ + from = ENSURE_RELPATH (from); + to = ENSURE_RELPATH (to); + if (renameat (basefd, from, basefd, to) == -1) + return -errno; + return 0; +} + +static int +callback_link (const char *from, const char *to) +{ + from = ENSURE_RELPATH (from); + to = ENSURE_RELPATH (to); + if (linkat (basefd, from, basefd, to, 0) == -1) + return -errno; + return 0; +} + +static int +can_write (const char *path) +{ + struct stat stbuf; + if (fstatat (basefd, path, &stbuf, 0) == -1) + { + if (errno == ENOENT) + return 0; + else + return -errno; + } + if (devino_set_contains (stbuf.st_dev, stbuf.st_ino)) + return -EROFS; + return 0; +} + +#define VERIFY_WRITE(path) do { \ + int r = can_write (path); \ + if (r != 0) \ + return r; \ + } while (0) + +static int +callback_chmod (const char *path, mode_t mode) +{ + path = ENSURE_RELPATH (path); + VERIFY_WRITE(path); + if (fchmodat (basefd, path, mode, 0) != 0) + return -errno; + return 0; +} + +static int +callback_chown (const char *path, uid_t uid, gid_t gid) +{ + path = ENSURE_RELPATH (path); + VERIFY_WRITE(path); + if (fchownat (basefd, path, uid, gid, 0) != 0) + return -errno; + return 0; +} + +static int +callback_truncate (const char *path, off_t size) +{ + glnx_fd_close int fd = -1; + + path = ENSURE_RELPATH (path); + VERIFY_WRITE(path); + + fd = openat (basefd, path, O_WRONLY); + if (fd == -1) + return -errno; + + if (ftruncate (fd, size) == -1) + return -errno; + + return 0; +} + +static int +callback_utime (const char *path, struct utimbuf *buf) +{ + struct timespec ts[2]; + + path = ENSURE_RELPATH (path); + + ts[0].tv_sec = buf->actime; + ts[0].tv_nsec = UTIME_OMIT; + ts[1].tv_sec = buf->modtime; + ts[1].tv_nsec = UTIME_OMIT; + + if (utimensat (basefd, path, ts, AT_SYMLINK_NOFOLLOW) == -1) + return -errno; + + return 0; +} + +static int +do_open (const char *path, mode_t mode, struct fuse_file_info *finfo) +{ + const int flags = finfo->flags & O_ACCMODE; + int fd; + struct stat stbuf; + + /* Support read only opens */ + G_STATIC_ASSERT (O_RDONLY == 0); + + path = ENSURE_RELPATH (path); + + if (flags == 0) + fd = openat (basefd, path, flags); + else + { + const int forced_excl_flags = flags | O_CREAT | O_EXCL; + /* Do an exclusive open, don't allow writable fds for existing + files */ + fd = openat (basefd, path, forced_excl_flags, mode); + /* If they didn't specify O_EXCL, give them EROFS if the file + * exists. + */ + if (fd == -1 && (flags & O_EXCL) == 0) + { + if (errno == EEXIST) + errno = EROFS; + } + else if (fd != -1) + { + if (fstat (fd, &stbuf) == -1) + return -errno; + devino_set_insert (stbuf.st_dev, stbuf.st_ino); + } + } + + if (fd == -1) + return -errno; + + finfo->fh = fd; + + return 0; +} + +static int +callback_open (const char *path, struct fuse_file_info *finfo) +{ + return do_open (path, 0, finfo); +} + +static int +callback_create(const char *path, mode_t mode, struct fuse_file_info *finfo) +{ + return do_open (path, mode, finfo); +} + +static int +callback_read (const char *path, char *buf, size_t size, off_t offset, + struct fuse_file_info *finfo) +{ + int r; + r = pread (finfo->fh, buf, size, offset); + if (r == -1) + return -errno; + return r; +} + +static int +callback_write (const char *path, const char *buf, size_t size, off_t offset, + struct fuse_file_info *finfo) +{ + int r; + r = pwrite (finfo->fh, buf, size, offset); + if (r == -1) + return -errno; + return r; +} + +static int +callback_statfs (const char *path, struct statvfs *st_buf) +{ + if (fstatvfs (basefd, st_buf) == -1) + return -errno; + return 0; +} + +static int +callback_release (const char *path, struct fuse_file_info *finfo) +{ + (void) close (finfo->fh); + return 0; +} + +static int +callback_fsync (const char *path, int crap, struct fuse_file_info *finfo) +{ + if (fsync (finfo->fh) == -1) + return -errno; + return 0; +} + +static int +callback_access (const char *path, int mode) +{ + path = ENSURE_RELPATH (path); + + /* Apparently at least GNU coreutils rm calls `faccessat(W_OK)` + * before trying to do an unlink. So...we'll just lie about + * writable access here. + */ + if (faccessat (basefd, path, mode, 0) == -1) + return -errno; + return 0; +} + +static int +callback_setxattr (const char *path, const char *name, const char *value, + size_t size, int flags) +{ + return -ENOTSUP; +} + +static int +callback_getxattr (const char *path, const char *name, char *value, + size_t size) +{ + return -ENOTSUP; +} + +/* + * List the supported extended attributes. + */ +static int +callback_listxattr (const char *path, char *list, size_t size) +{ + return -ENOTSUP; + +} + +/* + * Remove an extended attribute. + */ +static int +callback_removexattr (const char *path, const char *name) +{ + return -ENOTSUP; + +} + +struct fuse_operations callback_oper = { + .getattr = callback_getattr, + .readlink = callback_readlink, + .readdir = callback_readdir, + .mknod = callback_mknod, + .mkdir = callback_mkdir, + .symlink = callback_symlink, + .unlink = callback_unlink, + .rmdir = callback_rmdir, + .rename = callback_rename, + .link = callback_link, + .chmod = callback_chmod, + .chown = callback_chown, + .truncate = callback_truncate, + .utime = callback_utime, + .create = callback_create, + .open = callback_open, + .read = callback_read, + .write = callback_write, + .statfs = callback_statfs, + .release = callback_release, + .fsync = callback_fsync, + .access = callback_access, + + /* Extended attributes support for userland interaction */ + .setxattr = callback_setxattr, + .getxattr = callback_getxattr, + .listxattr = callback_listxattr, + .removexattr = callback_removexattr +}; + +enum +{ + KEY_HELP, + KEY_VERSION, +}; + +static void +usage (const char *progname) +{ + fprintf (stdout, + "usage: %s basepath mountpoint [options]\n" + "\n" + " Makes basepath visible at mountpoint such that files are read-only, directories are writable\n" + "\n" + "general options:\n" + " -o opt,[opt...] mount options\n" + " -h --help print help\n" + "\n", progname); +} + +static int +rofs_parse_opt (void *data, const char *arg, int key, + struct fuse_args *outargs) +{ + (void) data; + + switch (key) + { + case FUSE_OPT_KEY_NONOPT: + if (basefd == -1) + { + basefd = openat (AT_FDCWD, arg, O_RDONLY | O_NONBLOCK | O_DIRECTORY | O_CLOEXEC | O_NOCTTY); + if (basefd == -1) + { + perror ("openat"); + exit (1); + } + return 0; + } + else + { + return 1; + } + case FUSE_OPT_KEY_OPT: + return 1; + case KEY_HELP: + usage (outargs->argv[0]); + exit (0); + default: + fprintf (stderr, "see `%s -h' for usage\n", outargs->argv[0]); + exit (1); + } + return 1; +} + +static struct fuse_opt rofs_opts[] = { + FUSE_OPT_KEY ("-h", KEY_HELP), + FUSE_OPT_KEY ("--help", KEY_HELP), + FUSE_OPT_KEY ("-V", KEY_VERSION), + FUSE_OPT_KEY ("--version", KEY_VERSION), + FUSE_OPT_END +}; + +int +main (int argc, char *argv[]) +{ + struct fuse_args args = FUSE_ARGS_INIT (argc, argv); + int res; + + res = fuse_opt_parse (&args, &basefd, rofs_opts, rofs_parse_opt); + if (res != 0) + { + fprintf (stderr, "Invalid arguments\n"); + fprintf (stderr, "see `%s -h' for usage\n", argv[0]); + exit (1); + } + if (basefd == -1) + { + fprintf (stderr, "Missing basepath\n"); + fprintf (stderr, "see `%s -h' for usage\n", argv[0]); + exit (1); + } + + created_devino_hash = g_hash_table_new_full (devino_hash, devino_equal, g_free, NULL); + + fuse_main (args.argc, args.argv, &callback_oper, NULL); + + return 0; +} diff --git a/tests/admin-test.sh b/tests/admin-test.sh index edbf6651..c4644d3a 100755 --- a/tests/admin-test.sh +++ b/tests/admin-test.sh @@ -16,7 +16,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..10" @@ -155,6 +155,8 @@ assert_not_streq ${rev} ${newrev} assert_file_has_content sysroot/ostree/deploy/testos/deploy/${newrev}.0/etc/os-release 'NAME=TestOS' ${CMD_PREFIX} ostree admin status validate_bootloader +${CMD_PREFIX} ostree --repo=sysroot/ostree/repo refs testos:testos > reftest.txt +assert_file_has_content reftest.txt testos:buildmaster/x86_64-runtime echo "ok upgrade" diff --git a/tests/archive-test.sh b/tests/archive-test.sh index 9d9c0a28..e6f67cf5 100755 --- a/tests/archive-test.sh +++ b/tests/archive-test.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail $OSTREE checkout test2 checkout-test2 echo "ok checkout" diff --git a/tests/basic-test.sh b/tests/basic-test.sh index e8f1e9d2..c0487d64 100755 --- a/tests/basic-test.sh +++ b/tests/basic-test.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..48" @@ -387,6 +387,42 @@ assert_file_has_content test2-checkout/baz/cow moo assert_has_dir repo2/uncompressed-objects-cache echo "ok disable cache checkout" +# Whiteouts +cd ${test_tmpdir} +mkdir -p overlay/baz/ +touch overlay/baz/.wh.cow +touch overlay/.wh.deeper +touch overlay/anewfile +mkdir overlay/anewdir/ +touch overlay/anewdir/blah +$OSTREE --repo=repo commit -b overlay -s 'overlay' --tree=dir=overlay +rm overlay -rf + +for branch in test2 overlay; do + $OSTREE --repo=repo checkout --union --whiteouts ${branch} overlay-co +done +for f in .wh.deeper baz/cow baz/.wh.cow; do + assert_not_has_file overlay-co/${f} +done +assert_not_has_dir overlay-co/deeper +assert_has_file overlay-co/anewdir/blah +assert_has_file overlay-co/anewfile + +echo "ok whiteouts enabled" + +# Now double check whiteouts are not processed without --whiteouts +rm overlay-co -rf +for branch in test2 overlay; do + $OSTREE --repo=repo checkout --union ${branch} overlay-co +done +for f in .wh.deeper baz/cow baz/.wh.cow; do + assert_has_file overlay-co/${f} +done +assert_not_has_dir overlay-co/deeper +assert_has_file overlay-co/anewdir/blah +assert_has_file overlay-co/anewfile +echo "ok whiteouts disabled" + cd ${test_tmpdir} rm -rf test2-checkout mkdir -p test2-checkout diff --git a/tests/libtest.sh b/tests/libtest.sh index 885c6403..8cc4345f 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -22,6 +22,10 @@ test_tmpdir=$(pwd) export G_DEBUG=fatal-warnings +# Don't flag deployments as immutable so that test harnesses can +# easily clean up. +export OSTREE_SYSROOT_DEBUG=mutable-deployments + export TEST_GPG_KEYID_1="472CDAFA" export TEST_GPG_KEYID_2="CA950D41" export TEST_GPG_KEYID_3="DF444D67" @@ -33,11 +37,11 @@ cp -a ${SRCDIR}/gpghome ${test_tmpdir} export TEST_GPG_KEYHOME=${test_tmpdir}/gpghome export OSTREE_GPG_HOME=${test_tmpdir}/gpghome/trusted -if test -n "${OT_TESTS_DEBUG}"; then +if test -n "${OT_TESTS_DEBUG:-}"; then set -x fi -if test -n "$OT_TESTS_VALGRIND"; then +if test -n "${OT_TESTS_VALGRIND:-}"; then CMD_PREFIX="env G_SLICE=always-malloc valgrind -q --leak-check=full --num-callers=30 --suppressions=${SRCDIR}/ostree-valgrind.supp" else CMD_PREFIX="env LD_PRELOAD=${SRCDIR}/libreaddir-rand.so" @@ -139,8 +143,8 @@ setup_test_repository () { setup_fake_remote_repo1() { mode=$1 - commit_opts=$2 - args=$3 + commit_opts=${2:-} + args=${3:-} shift oldpwd=`pwd` mkdir ostree-srv @@ -272,7 +276,7 @@ EOF mkdir ${test_tmpdir}/httpd cd httpd ln -s ${test_tmpdir} ostree - ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port $args + ${CMD_PREFIX} ostree trivial-httpd --autoexit --daemonize -p ${test_tmpdir}/httpd-port port=$(cat ${test_tmpdir}/httpd-port) echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address cd ${oldpwd} @@ -280,15 +284,9 @@ EOF os_repository_new_commit () { - boot_checksum_iteration=$1 - content_iteration=$2 + boot_checksum_iteration=${1:-0} + content_iteration=${2:-0} echo "BOOT ITERATION: $boot_checksum_iteration" - if test -z "$boot_checksum_iteration"; then - boot_checksum_iteration=0 - fi - if test -z "$content_iteration"; then - content_iteration=0 - fi cd ${test_tmpdir}/osdata rm boot/* echo "new: a kernel ${boot_checksum_iteration}" > boot/vmlinuz-3.6.0 diff --git a/tests/pre-endian-deltas-repo-big.tar.xz b/tests/pre-endian-deltas-repo-big.tar.xz new file mode 100644 index 00000000..05e51d69 Binary files /dev/null and b/tests/pre-endian-deltas-repo-big.tar.xz differ diff --git a/tests/pre-endian-deltas-repo-little.tar.xz b/tests/pre-endian-deltas-repo-little.tar.xz new file mode 100644 index 00000000..ef05045c Binary files /dev/null and b/tests/pre-endian-deltas-repo-little.tar.xz differ diff --git a/tests/pull-test.sh b/tests/pull-test.sh index 42979fe9..9c8b41fa 100755 --- a/tests/pull-test.sh +++ b/tests/pull-test.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail function repo_init() { cd ${test_tmpdir} @@ -72,6 +72,21 @@ $OSTREE show --print-detached-metadata-key=SIGNATURE main > main-meta assert_file_has_content main-meta "HANCOCK" echo "ok pull detached metadata" +cd ${test_tmpdir} +mkdir parentpullrepo +${CMD_PREFIX} ostree --repo=parentpullrepo init --mode=archive-z2 +${CMD_PREFIX} ostree --repo=parentpullrepo remote add --set=gpg-verify=false origin file://$(pwd)/ostree-srv/gnomerepo +parent_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) +rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) +${CMD_PREFIX} ostree --repo=parentpullrepo pull origin main@${parent_rev} +${CMD_PREFIX} ostree --repo=parentpullrepo rev-parse origin:main > main.txt +assert_file_has_content main.txt ${parent_rev} +${CMD_PREFIX} ostree --repo=parentpullrepo fsck +${CMD_PREFIX} ostree --repo=parentpullrepo pull origin main +${CMD_PREFIX} ostree --repo=parentpullrepo rev-parse origin:main > main.txt +assert_file_has_content main.txt ${rev} +echo "ok pull specific commit" + cd ${test_tmpdir} repo_init ${CMD_PREFIX} ostree --repo=repo pull origin main @@ -91,12 +106,30 @@ cd .. rm main-files -rf # Generate delta that we'll use ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main +prev_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main^) +new_rev=$(ostree --repo=ostree-srv/gnomerepo rev-parse main) +ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} -${CMD_PREFIX} ostree --repo=repo pull origin main +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +${CMD_PREFIX} ostree --repo=repo pull --dry-run --require-static-deltas origin main >dry-run-pull.txt +assert_file_has_content dry-run-pull.txt 'Delta update: 0/1 parts' +rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) +assert_streq "${prev_rev}" "${rev}" ${CMD_PREFIX} ostree --repo=repo fsck cd ${test_tmpdir} +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main +rev=$(${CMD_PREFIX} ostree --repo=repo rev-parse origin:main) +assert_streq "${new_rev}" "${rev}" +${CMD_PREFIX} ostree --repo=repo fsck + +cd ${test_tmpdir} +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} ${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas origin main ${CMD_PREFIX} ostree --repo=repo fsck @@ -109,6 +142,33 @@ assert_not_has_file baz/saucer echo "ok static delta" +cd ${test_tmpdir} +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --swap-endianness main +${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo summary -u + +repo_init +${CMD_PREFIX} ostree --repo=repo pull origin main@${prev_rev} +${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas --dry-run origin main >byteswapped-dry-run-pull.txt +${CMD_PREFIX} ostree --repo=repo fsck + +if ! diff -u dry-run-pull.txt byteswapped-dry-run-pull.txt; then + assert_not_reached "byteswapped delta differs in size" +fi + +echo "ok pull byteswapped delta" + +cd ${test_tmpdir} +rm ostree-srv/gnomerepo/deltas -rf +ostree --repo=ostree-srv/gnomerepo summary -u +repo_init +if ${CMD_PREFIX} ostree --repo=repo pull --require-static-deltas origin main 2>err.txt; then + assert_not_reached "--require-static-deltas unexpectedly succeeded" +fi +assert_file_has_content err.txt "deltas required, but none found" +${CMD_PREFIX} ostree --repo=repo fsck + +echo "ok delta required but don't exist" + cd ${test_tmpdir} rm main-files -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo checkout main main-files @@ -119,6 +179,7 @@ cd .. rm main-files -rf # Generate new delta that we'll use ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate --inline main +ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=repo pull origin main @@ -142,6 +203,7 @@ ${CMD_PREFIX} ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo commit -b main - cd .. rm main-files -rf ${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo static-delta generate main +ostree --repo=ostree-srv/gnomerepo summary -u cd ${test_tmpdir} ${CMD_PREFIX} ostree --repo=repo pull origin main diff --git a/tests/test-admin-deploy-2.sh b/tests/test-admin-deploy-2.sh index d6117de0..ef6b5953 100755 --- a/tests/test-admin-deploy-2.sh +++ b/tests/test-admin-deploy-2.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh @@ -49,7 +49,6 @@ os_repository_new_commit "1" bootcsum3=${bootcsum} ${CMD_PREFIX} ostree admin upgrade --os=testos -rev=${newrev} newrev=$(${CMD_PREFIX} ostree --repo=sysroot/ostree/repo rev-parse testos/buildmaster/x86_64-runtime) assert_not_streq ${rev} ${newrev} assert_not_streq ${bootcsum1} ${bootcsum2} diff --git a/tests/test-admin-deploy-clean.sh b/tests/test-admin-deploy-clean.sh index 19f71e61..58283108 100644 --- a/tests/test-admin-deploy-clean.sh +++ b/tests/test-admin-deploy-clean.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-etcmerge-cornercases.sh b/tests/test-admin-deploy-etcmerge-cornercases.sh index 0541a670..4b0d781b 100644 --- a/tests/test-admin-deploy-etcmerge-cornercases.sh +++ b/tests/test-admin-deploy-etcmerge-cornercases.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-grub2.sh b/tests/test-admin-deploy-grub2.sh index 94c4bc03..8da294d7 100755 --- a/tests/test-admin-deploy-grub2.sh +++ b/tests/test-admin-deploy-grub2.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-karg.sh b/tests/test-admin-deploy-karg.sh index 9decec67..a8c1e594 100644 --- a/tests/test-admin-deploy-karg.sh +++ b/tests/test-admin-deploy-karg.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-switch.sh b/tests/test-admin-deploy-switch.sh index 7e1e173a..4a52000c 100755 --- a/tests/test-admin-deploy-switch.sh +++ b/tests/test-admin-deploy-switch.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-syslinux.sh b/tests/test-admin-deploy-syslinux.sh index 97ee32df..5883f76d 100755 --- a/tests/test-admin-deploy-syslinux.sh +++ b/tests/test-admin-deploy-syslinux.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-deploy-uboot.sh b/tests/test-admin-deploy-uboot.sh index 219db14b..c22af6f3 100755 --- a/tests/test-admin-deploy-uboot.sh +++ b/tests/test-admin-deploy-uboot.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-instutil-set-kargs.sh b/tests/test-admin-instutil-set-kargs.sh index 04f98c01..33b2b74e 100644 --- a/tests/test-admin-instutil-set-kargs.sh +++ b/tests/test-admin-instutil-set-kargs.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-locking.sh b/tests/test-admin-locking.sh index 6b432202..5f00f571 100644 --- a/tests/test-admin-locking.sh +++ b/tests/test-admin-locking.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-pull-deploy-commit.sh b/tests/test-admin-pull-deploy-commit.sh index 08c1e6bd..e1f7def6 100644 --- a/tests/test-admin-pull-deploy-commit.sh +++ b/tests/test-admin-pull-deploy-commit.sh @@ -19,7 +19,7 @@ # See https://github.com/GNOME/ostree/pull/145 -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-upgrade-not-backwards.sh b/tests/test-admin-upgrade-not-backwards.sh index c42dc242..1b99e25d 100644 --- a/tests/test-admin-upgrade-not-backwards.sh +++ b/tests/test-admin-upgrade-not-backwards.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-admin-upgrade-unconfigured.sh b/tests/test-admin-upgrade-unconfigured.sh index cbc1e753..38df710d 100644 --- a/tests/test-admin-upgrade-unconfigured.sh +++ b/tests/test-admin-upgrade-unconfigured.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-archivez.sh b/tests/test-archivez.sh index d5b7fc35..5db973ff 100755 --- a/tests/test-archivez.sh +++ b/tests/test-archivez.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-auto-summary.sh b/tests/test-auto-summary.sh index 2452e553..6cb52e8c 100755 --- a/tests/test-auto-summary.sh +++ b/tests/test-auto-summary.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-basic-user.sh b/tests/test-basic-user.sh index c1705254..f53de89a 100755 --- a/tests/test-basic-user.sh +++ b/tests/test-basic-user.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-basic.sh b/tests/test-basic.sh index 3c55756e..ae55aab2 100755 --- a/tests/test-basic.sh +++ b/tests/test-basic.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-commit-sign.sh b/tests/test-commit-sign.sh index 0dfbb768..2db671ee 100755 --- a/tests/test-commit-sign.sh +++ b/tests/test-commit-sign.sh @@ -17,9 +17,9 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail -if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme'; then +if ! ostree --version | grep -q -e '\+gpgme'; then exit 77 fi diff --git a/tests/test-corruption.sh b/tests/test-corruption.sh index 9ad6aaf2..ef0e94ef 100755 --- a/tests/test-corruption.sh +++ b/tests/test-corruption.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..2" diff --git a/tests/test-delta.sh b/tests/test-delta.sh index 2dddefee..ebe35571 100755 --- a/tests/test-delta.sh +++ b/tests/test-delta.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh @@ -114,6 +114,41 @@ fi echo 'ok generate' +${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show.txt +assert_file_has_content show.txt 'Endianness: \(little\|big\)' + +echo 'ok show' + +${CMD_PREFIX} ostree --repo=repo static-delta generate --swap-endianness --from=${origrev} --to=${newrev} +${CMD_PREFIX} ostree --repo=repo static-delta show ${origrev}-${newrev} > show-swapped.txt +totalsize_orig=$(grep 'Total Size:' show.txt) +totalsize_swapped=$(grep 'Total Size:' show-swapped.txt) +assert_not_streq "${totalsize_orig}" "" +assert_streq "${totalsize_orig}" "${totalsize_swapped}" + +echo 'ok generate + show endian swapped' + +tar xf ${SRCDIR}/pre-endian-deltas-repo-big.tar.xz +mv pre-endian-deltas-repo{,-big} +tar xf ${SRCDIR}/pre-endian-deltas-repo-little.tar.xz +mv pre-endian-deltas-repo{,-little} +legacy_origrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main^) +legacy_newrev=$(${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big rev-parse main) +${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big static-delta show ${legacy_origrev}-${legacy_newrev} > show-legacy-big.txt +totalsize_legacy_big=$(grep 'Total Size:' show-legacy-big.txt) +${CMD_PREFIX} ostree --repo=pre-endian-deltas-repo-big static-delta show ${legacy_origrev}-${legacy_newrev} > show-legacy-little.txt +totalsize_legacy_little=$(grep 'Total Size:' show-legacy-little.txt) +for f in show-legacy-{big,little}.txt; do + if grep 'Endianness:.*heuristic' $f; then + found_heuristic=yes + break + fi +done +assert_streq "${found_heuristic}" "yes" +assert_streq "${totalsize_legacy_big}" "${totalsize_legacy_little}" + +echo 'ok heuristic endian detection' + mkdir repo2 && ${CMD_PREFIX} ostree --repo=repo2 init --mode=archive-z2 ${CMD_PREFIX} ostree --repo=repo2 pull-local repo ${newrev} ${CMD_PREFIX} ostree --repo=repo2 fsck diff --git a/tests/test-export.sh b/tests/test-export.sh new file mode 100755 index 00000000..18f2f7ae --- /dev/null +++ b/tests/test-export.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +. $(dirname $0)/libtest.sh + +setup_test_repository "archive-z2" + +echo '1..2' + +$OSTREE checkout test2 test2-co +$OSTREE commit --no-xattrs -b test2-noxattrs -s "test2 without xattrs" --tree=dir=test2-co +rm test2-co -rf + +cd ${test_tmpdir} +${OSTREE} 'export' test2-noxattrs -o test2.tar +mkdir t +(cd t && tar xf ../test2.tar) +ostree --repo=repo diff --no-xattrs test2-noxattrs ./t > diff.txt +assert_file_empty diff.txt +rm test2.tar diff.txt t -rf + +echo 'ok export gnutar diff (no xattrs)' + +cd ${test_tmpdir} +${OSTREE} 'export' test2 -o test2.tar +${OSTREE} commit -b test2-from-tar -s 'Import from tar' --tree=tar=test2.tar +ostree --repo=repo diff test2 test2-from-tar +assert_file_empty diff.txt +rm test2.tar diff.txt t -rf + +echo 'ok export import' + diff --git a/tests/test-gpg-signed-commit.sh b/tests/test-gpg-signed-commit.sh index ba361c7a..b713da2d 100644 --- a/tests/test-gpg-signed-commit.sh +++ b/tests/test-gpg-signed-commit.sh @@ -18,9 +18,9 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail -if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+gpgme'; then +if ! ostree --version | grep -q -e '\+gpgme'; then exit 77 fi diff --git a/tests/test-help.sh b/tests/test-help.sh index 37d9fa1d..ca555b1b 100755 --- a/tests/test-help.sh +++ b/tests/test-help.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh @@ -45,7 +45,7 @@ test_recursive() { if [ $? = 0 ] ; then echo 1>&2 "missing subcommand but 0 exit status"; exit 1 fi - set -e + set -euo pipefail # error message and usage goes to standard error assert_file_has_content err "[Uu]sage" assert_file_has_content err "$cmd" diff --git a/tests/test-libarchive-import.c b/tests/test-libarchive-import.c new file mode 100644 index 00000000..928b1491 --- /dev/null +++ b/tests/test-libarchive-import.c @@ -0,0 +1,256 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" +#include "libglnx.h" +#include +#include +#include +#include + +#include +#include +#include + +typedef struct { + OstreeRepo *repo; + int fd; + int fd_empty; + char *tmpd; +} TestData; + +static void +test_data_init (TestData *td) +{ + GError *error = NULL; + struct archive *a = archive_write_new (); + struct archive_entry *ae; + + td->tmpd = g_mkdtemp (g_strdup ("/var/tmp/test-libarchive-import-XXXXXX")); + g_assert_cmpint (0, ==, chdir (td->tmpd)); + + td->fd = openat (AT_FDCWD, "foo.tar.gz", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0644); + (void) unlink ("foo.tar.gz"); + + g_assert_no_error (error); + g_assert (td->fd >= 0); + + g_assert_cmpint (0, ==, archive_write_set_format_pax (a)); + g_assert_cmpint (0, ==, archive_write_add_filter_gzip (a)); + g_assert_cmpint (0, ==, archive_write_open_fd (a, td->fd)); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/"); + archive_entry_set_mode (ae, S_IFDIR | 0755); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/file"); + archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_size (ae, 4); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + g_assert_cmpint (4, ==, archive_write_data (a, "foo\n", 4)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/devnull"); + archive_entry_set_mode (ae, S_IFCHR | 0777); + archive_entry_set_devmajor (ae, 1); + archive_entry_set_devminor (ae, 3); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + archive_entry_free (ae); + + ae = archive_entry_new (); + archive_entry_set_pathname (ae, "/anotherfile"); + archive_entry_set_mode (ae, S_IFREG | 0777); + archive_entry_set_size (ae, 4); + g_assert_cmpint (0, ==, archive_write_header (a, ae)); + g_assert_cmpint (4, ==, archive_write_data (a, "bar\n", 4)); + archive_entry_free (ae); + + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_close (a)); + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_free (a)); + + td->fd_empty = openat (AT_FDCWD, "empty.tar.gz", O_CREAT | O_EXCL | O_RDWR | O_CLOEXEC, 0644); + g_assert (td->fd_empty >= 0); + (void) unlink ("empty.tar.gz"); + + a = archive_write_new (); + g_assert (a); + + g_assert_cmpint (0, ==, archive_write_set_format_pax (a)); + g_assert_cmpint (0, ==, archive_write_add_filter_gzip (a)); + g_assert_cmpint (0, ==, archive_write_open_fd (a, td->fd_empty)); + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_close (a)); + g_assert_cmpint (ARCHIVE_OK, ==, archive_write_free (a)); + + { g_autoptr(GFile) repopath = g_file_new_for_path ("repo"); + td->repo = ostree_repo_new (repopath); + + g_assert_cmpint (0, ==, mkdir ("repo", 0755)); + + ostree_repo_create (td->repo, OSTREE_REPO_MODE_BARE_USER, NULL, &error); + g_assert_no_error (error); + } +} + +static gboolean +spawn_cmdline (const char *cmd, GError **error) +{ + int estatus; + if (!g_spawn_command_line_sync (cmd, NULL, NULL, &estatus, error)) + return FALSE; + if (!g_spawn_check_exit_status (estatus, error)) + return FALSE; + return TRUE; +} + +static void +test_libarchive_noautocreate_empty (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + g_assert_cmpint (0, ==, lseek (td->fd_empty, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd_empty, 8192)); + + (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); + g_assert_no_error (error); + g_assert (ostree_mutable_tree_get_metadata_checksum (mtree) == NULL); +} + +static void +test_libarchive_autocreate_empty (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + opts.autocreate_parents = 1; + + g_assert_cmpint (0, ==, lseek (td->fd_empty, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd_empty, 8192)); + + (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); + g_assert_no_error (error); + g_assert (ostree_mutable_tree_get_metadata_checksum (mtree) != NULL); +} + +static void +test_libarchive_error_device_file (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + + g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + + (void)ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error); + g_assert (error != NULL); +} + +static void +test_libarchive_ignore_device_file (gconstpointer data) +{ + TestData *td = (void*)data; + GError *error = NULL; + GCancellable *cancellable = NULL; + struct archive *a = archive_read_new (); + OstreeRepoImportArchiveOptions opts = { 0, }; + glnx_unref_object OstreeMutableTree *mtree = ostree_mutable_tree_new (); + glnx_unref_object GFile *root = NULL; + g_autofree char *commit_checksum = NULL; + + g_assert_cmpint (0, ==, lseek (td->fd, 0, SEEK_SET)); + g_assert_cmpint (0, ==, archive_read_support_format_all (a)); + g_assert_cmpint (0, ==, archive_read_support_filter_all (a)); + g_assert_cmpint (0, ==, archive_read_open_fd (a, td->fd, 8192)); + + opts.ignore_unsupported_content = TRUE; + + if (!ostree_repo_prepare_transaction (td->repo, NULL, cancellable, &error)) + goto out; + + if (!ostree_repo_import_archive_to_mtree (td->repo, &opts, a, mtree, NULL, NULL, &error)) + goto out; + + if (!ostree_repo_write_mtree (td->repo, mtree, &root, cancellable, &error)) + goto out; + + if (!ostree_repo_write_commit (td->repo, NULL, "", "", NULL, + OSTREE_REPO_FILE (root), + &commit_checksum, cancellable, &error)) + goto out; + + ostree_repo_transaction_set_ref (td->repo, NULL, "foo", commit_checksum); + + if (!ostree_repo_commit_transaction (td->repo, NULL, cancellable, &error)) + goto out; + + if (!spawn_cmdline ("ostree --repo=repo ls foo file", &error)) + goto out; + + if (!spawn_cmdline ("ostree --repo=repo ls foo anotherfile", &error)) + goto out; + + if (spawn_cmdline ("ostree --repo=repo ls foo devnull", &error)) + g_assert_not_reached (); + g_assert (error != NULL); + g_clear_error (&error); + + out: + g_assert_no_error (error); +} + +int main (int argc, char **argv) +{ + TestData td = {NULL,}; + int r; + + test_data_init (&td); + + g_test_init (&argc, &argv, NULL); + + g_test_add_data_func ("/libarchive/noautocreate-empty", &td, test_libarchive_noautocreate_empty); + g_test_add_data_func ("/libarchive/autocreate-empty", &td, test_libarchive_autocreate_empty); + g_test_add_data_func ("/libarchive/error-device-file", &td, test_libarchive_error_device_file); + g_test_add_data_func ("/libarchive/ignore-device-file", &td, test_libarchive_ignore_device_file); + + r = g_test_run(); + + if (td.tmpd) + (void) glnx_shutil_rm_rf_at (AT_FDCWD, td.tmpd, NULL, NULL); + return r; +} diff --git a/tests/test-libarchive.sh b/tests/test-libarchive.sh index c875f6b0..92e24083 100755 --- a/tests/test-libarchive.sh +++ b/tests/test-libarchive.sh @@ -17,9 +17,9 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail -if ! ${CMD_PREFIX} ostree --version | grep -q -e '\+libarchive'; then +if ! ostree --version | grep -q -e '\+libarchive'; then exit 77 fi diff --git a/tests/test-local-pull-depth.sh b/tests/test-local-pull-depth.sh index 4f8988dc..e89d0914 100755 --- a/tests/test-local-pull-depth.sh +++ b/tests/test-local-pull-depth.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-local-pull.sh b/tests/test-local-pull.sh index df5342de..a9beb083 100755 --- a/tests/test-local-pull.sh +++ b/tests/test-local-pull.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-oldstyle-partial.sh b/tests/test-oldstyle-partial.sh index b7da9a86..220b0831 100644 --- a/tests/test-oldstyle-partial.sh +++ b/tests/test-oldstyle-partial.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-prune.sh b/tests/test-prune.sh index 28695b8f..a1322d90 100644 --- a/tests/test-prune.sh +++ b/tests/test-prune.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-archive-z.sh b/tests/test-pull-archive-z.sh index 0d208528..6482f6f9 100755 --- a/tests/test-pull-archive-z.sh +++ b/tests/test-pull-archive-z.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-commit-only.sh b/tests/test-pull-commit-only.sh index 136cc3b9..775b2f71 100755 --- a/tests/test-pull-commit-only.sh +++ b/tests/test-pull-commit-only.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-corruption.sh b/tests/test-pull-corruption.sh index e81e4229..1df31943 100755 --- a/tests/test-pull-corruption.sh +++ b/tests/test-pull-corruption.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh @@ -55,4 +55,4 @@ gjs --help >/dev/null 2>&1 || exit 77 gjs $(dirname $0)/corrupt-repo-ref.js ${repopath} main || true assert_file_has_content corrupted-status.txt 'Changed byte' do_corrupt_pull_test -echo "ok corruption $iteration" +echo "ok corruption" diff --git a/tests/test-pull-depth.sh b/tests/test-pull-depth.sh index 35191cc4..7a52f9cb 100644 --- a/tests/test-pull-depth.sh +++ b/tests/test-pull-depth.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-large-metadata.sh b/tests/test-pull-large-metadata.sh index f86ce4be..c50d7943 100644 --- a/tests/test-pull-large-metadata.sh +++ b/tests/test-pull-large-metadata.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh index 84ef3739..52d2d503 100755 --- a/tests/test-pull-metalink.sh +++ b/tests/test-pull-metalink.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-mirror-summary.sh b/tests/test-pull-mirror-summary.sh index 958044f9..de55b59b 100755 --- a/tests/test-pull-mirror-summary.sh +++ b/tests/test-pull-mirror-summary.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-resume.sh b/tests/test-pull-resume.sh index 25ee907e..1e7220d0 100755 --- a/tests/test-pull-resume.sh +++ b/tests/test-pull-resume.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-subpath.sh b/tests/test-pull-subpath.sh index f2abbb3b..70348322 100644 --- a/tests/test-pull-subpath.sh +++ b/tests/test-pull-subpath.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-pull-summary-sigs.sh b/tests/test-pull-summary-sigs.sh index 7afca9a3..dbcc67a5 100644 --- a/tests/test-pull-summary-sigs.sh +++ b/tests/test-pull-summary-sigs.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-remote-add.sh b/tests/test-remote-add.sh index 392dda54..2294a06d 100755 --- a/tests/test-remote-add.sh +++ b/tests/test-remote-add.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-remote-gpg-import.sh b/tests/test-remote-gpg-import.sh index 0dc5424e..fc833493 100755 --- a/tests/test-remote-gpg-import.sh +++ b/tests/test-remote-gpg-import.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-repo-checkout-subpath.sh b/tests/test-repo-checkout-subpath.sh index 343b2614..bf792184 100755 --- a/tests/test-repo-checkout-subpath.sh +++ b/tests/test-repo-checkout-subpath.sh @@ -18,7 +18,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail . $(dirname $0)/libtest.sh diff --git a/tests/test-reset-nonlinear.sh b/tests/test-reset-nonlinear.sh index 1d0f8de6..735f1523 100755 --- a/tests/test-reset-nonlinear.sh +++ b/tests/test-reset-nonlinear.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-rofiles-fuse.sh b/tests/test-rofiles-fuse.sh new file mode 100755 index 00000000..24ee2648 --- /dev/null +++ b/tests/test-rofiles-fuse.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# +# Copyright (C) 2016 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -euo pipefail + +echo "1..5" + +. $(dirname $0)/libtest.sh +setup_test_repository "bare-user" + +mkdir mnt + +$OSTREE checkout test2 checkout-test2 + +rofiles-fuse checkout-test2 mnt +cleanup_fuse() { + fusermount -u ${test_tmpdir}/mnt || true +} +trap cleanup_fuse EXIT +assert_file_has_content mnt/firstfile first +echo "ok mount" + +if cp /dev/null mnt/firstfile 2>err.txt; then + assert_not_reached "inplace mutation" +fi +assert_file_has_content err.txt "Read-only file system" +assert_file_has_content mnt/firstfile first +assert_file_has_content checkout-test2/firstfile first + +echo "ok failed inplace mutation" + +echo anewfile-for-fuse > mnt/anewfile-for-fuse +assert_file_has_content mnt/anewfile-for-fuse anewfile-for-fuse +assert_file_has_content checkout-test2/anewfile-for-fuse anewfile-for-fuse + +mkdir mnt/newfusedir +for i in $(seq 5); do + echo ${i}-morenewfuse-${i} > mnt/newfusedir/test-morenewfuse.${i} +done +assert_file_has_content checkout-test2/newfusedir/test-morenewfuse.3 3-morenewfuse-3 + +echo "ok new content" + +rm mnt/baz/cow +assert_not_has_file checkout-test2/baz/cow +rm mnt/baz/another -rf +assert_not_has_dir checkout-test2/baz/another + +echo "ok deletion" + +ostree --repo=repo commit -b test2 -s fromfuse --link-checkout-speedup --tree=dir=checkout-test2 + +echo "ok commit" + + + + + diff --git a/tests/test-setuid.sh b/tests/test-setuid.sh index 02aa9f61..5354d1f3 100755 --- a/tests/test-setuid.sh +++ b/tests/test-setuid.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail echo "1..1" diff --git a/tests/test-xattrs.sh b/tests/test-xattrs.sh index b95707e2..6a83a0bc 100755 --- a/tests/test-xattrs.sh +++ b/tests/test-xattrs.sh @@ -17,7 +17,7 @@ # Free Software Foundation, Inc., 59 Temple Place - Suite 330, # Boston, MA 02111-1307, USA. -set -e +set -euo pipefail touch test-xattrs if ! setfattr -n user.testvalue -v somevalue test-xattrs; then