packaging/ostree
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New upstream version 2016.11

Browse Source
This commit is contained in:
Simon McVittie committed 2016-10-07 21:52:21 +01:00
2 parents 9732c6c712 a0e1344cf8
commit f04805f646
179 files changed
+79036 -7233
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/libostree/README.md
-81
View File
@@ -1,81 +0,0 @@
Repository design
-----------------
At the heart of OSTree is the repository. It's very similar to git,
with the idea of content-addressed storage. However, OSTree is
designed to store operating system binaries, not source code. There
are several consequences to this. The key difference as compared to
git is that the OSTree definition of "content" includes key Unix
metadata such as owner uid/gid, as well as all extended attributes.
Essentially OSTree is designed so that if two files have the same
OSTree checksum, it's safe to replace them with a hard link. This
fundamental design means that an OSTree repository imposes negligible
overhead. In contrast, a git repository stores copies of
zlib-compressed data.
Key differences versus git
--------------------------
* As mentioned above, extended attributes and owner uid/gid are versioned
* Optimized for Unix hardlinks between repository and checkout
* SHA256 instead of SHA1
* Support for empty directories
Binary files
------------
While this is still in planning, I plan to heavily optimize OSTree for
versioning ELF operating systems. In industry jargon, this would be
"content-aware storage".
Trimming history
----------------
OSTree will also be optimized to trim intermediate history; in theory
one can regenerate binaries from corresponding (git) source code, so
we don't need to keep all possible builds over time.
MILESTONE 1
-----------
* Basic pack files (like git)
MILESTONE 2
-----------
* Store checksums as ay
* Drop version/metadata from tree/dirmeta objects
* Add index size to superindex, pack size to index
- So pull can calculate how much we need to download
* Split pack files into metadata/data
* pull: Extract all we can from each packfile one at a time, then delete it
* Restructure repository so that links can be generated as a cache;
i.e. objects/raw, pack files are now the canonical
* For files, checksum combination of metadata variant + raw data
- i.e. there is only OSTREE_OBJECT_TYPE_FILE (again)
MILESTONE 3
-----------
* Drop archive/raw distinction - archive repositories always generate
packfiles per commit
* Include git packv4 ideas:
- metadata packfiles have string dictionary (tree filenames and checksums)
- data packfiles match up similar objects
* Rolling checksums for partitioning large files? Kernel debuginfo
* Improved pack clustering
- file fingerprinting?
* ELF-x86 aware deltas
Related work in storage
-----------------------
git: http://git-scm.com/
Venti: http://plan9.bell-labs.com/magic/man2html/6/venti
Elephant FS: http://www.hpl.hp.com/personal/Alistair_Veitch/papers/elephant-hotos/index.html
Compression
-----------
xdelta: http://xdelta.org/
Bsdiff: http://www.daemonology.net/bsdiff/
xz: http://tukaani.org/xz/
src/libostree/libostree.sym
+2 -1
View File
@@ -355,6 +355,7 @@ global:
/* No new symbols in 2016.9 */
/* No new symbols in 2016.10 */
/* No new symbols in 2016.11 */
/* NOTE NOTE NOTE
* Versions above here are released. Only add symbols below this line.
@@ -362,7 +363,7 @@ global:
*/
/* Remove comment when first new symbol is added
LIBOSTREE_2016.11
LIBOSTREE_2016.12
global:
someostree_symbol_deleteme;
} LIBOSTREE_2016.8;
src/libostree/ostree-core-private.h
+3 -2
View File
@@ -121,10 +121,11 @@ static inline char * _ostree_get_commitpartial_path (const char *checksum)
return g_strconcat ("state/", checksum, ".commitpartial", NULL);
}
void
gboolean
_ostree_parse_delta_name (const char *delta_name,
char **out_from,
char **out_to);
char **out_to,
GError **error);
void
_ostree_loose_path (char *buf,
src/libostree/ostree-core.c
+20 -4
View File
@@ -1584,12 +1584,26 @@ _ostree_get_relative_static_delta_part_path (const char *from,
return _ostree_get_relative_static_delta_path (from, to, partstr);
}
void
_ostree_parse_delta_name (const char *delta_name,
gboolean
_ostree_parse_delta_name (const char *delta_name,
char **out_from,
char **out_to)
char **out_to,
GError **error)
{
g_auto(GStrv) parts = g_strsplit (delta_name, "-", 2);
g_auto(GStrv) parts = NULL;
g_return_val_if_fail (delta_name != NULL, FALSE);
parts = g_strsplit (delta_name, "-", 2);
/* NB: if delta_name is "", parts[0] is NULL, but the error
* validate_checksum_string() gives for "" is nice enough,
* so we just coerce it here */
if (!ostree_validate_checksum_string (parts[0] ?: "", error))
return FALSE;
if (parts[0] && parts[1] &&
!ostree_validate_checksum_string (parts[1], error))
return FALSE;
*out_from = *out_to = NULL;
if (parts[0] && parts[1])
@@ -1601,6 +1615,8 @@ _ostree_parse_delta_name (const char *delta_name,
{
ot_transfer_out_value (out_to, &parts[0]);
}
return TRUE;
}
/*
src/libostree/ostree-enumtypes.c
+56
View File
@@ -0,0 +1,56 @@
/* Generated data (by glib-mkenums) */
/*
* Copyright (C) 2015 Red Hat, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#include "ostree-enumtypes.h"
/* enumerations from "./src/libostree/ostree-fetcher.h" */
#include "./src/libostree/ostree-fetcher.h"
GType
_ostree_fetcher_config_flags_get_type (void)
{
static volatile gsize the_type__volatile = 0;
if (g_once_init_enter (&the_type__volatile))
{
static const GFlagsValue values[] = {
{ OSTREE_FETCHER_FLAGS_NONE,
"OSTREE_FETCHER_FLAGS_NONE",
"none" },
{ OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE,
"OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE",
"tls-permissive" },
{ 0, NULL, NULL }
};
GType the_type = g_flags_register_static (
g_intern_static_string ("OstreeFetcherConfigFlags"),
values);
g_once_init_leave (&the_type__volatile, the_type);
}
return the_type__volatile;
}
/* Generated data ends here */
src/libostree/ostree-enumtypes.h
+38
View File
@@ -0,0 +1,38 @@
/* Generated data (by glib-mkenums) */
/*
* Copyright (C) 2015 Red Hat, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#pragma once
#include <glib-object.h>
G_BEGIN_DECLS
/* Enumerations from "./src/libostree/ostree-fetcher.h" */
#define OSTREE_TYPE_FETCHER_CONFIG_FLAGS (_ostree_fetcher_config_flags_get_type ())
GType _ostree_fetcher_config_flags_get_type (void) G_GNUC_CONST;
G_END_DECLS
/* Generated data ends here */
src/libostree/ostree-repo-checkout.c
+2 -2
View File
@@ -106,7 +106,7 @@ static gboolean
fsync_is_enabled (OstreeRepo *self,
OstreeRepoCheckoutAtOptions *options)
{
return options->enable_fsync || !self->disable_fsync;
return options->enable_fsync;
}
static gboolean
@@ -860,7 +860,7 @@ ostree_repo_checkout_tree_at (OstreeRepo *self,
new_opts.mode = options->mode;
new_opts.overwrite_mode = options->overwrite_mode;
new_opts.enable_uncompressed_cache = options->enable_uncompressed_cache;
new_opts.enable_fsync = !options->disable_fsync;
new_opts.enable_fsync = options->disable_fsync ? FALSE : self->disable_fsync;
new_opts.process_whiteouts = options->process_whiteouts;
new_opts.no_copy_fallback = options->no_copy_fallback;
new_opts.subpath = options->subpath;
src/libostree/ostree-repo-pull.c
+4 -1
View File
@@ -2272,6 +2272,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
const char *url_override = NULL;
g_autofree char *base_meta_url = NULL;
g_autofree char *base_content_url = NULL;
gboolean mirroring_into_archive;
if (options)
{
@@ -2312,6 +2313,8 @@ ostree_repo_pull_with_options (OstreeRepo *self,
pull_data->is_untrusted = (flags & OSTREE_REPO_PULL_FLAGS_UNTRUSTED) > 0;
pull_data->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
mirroring_into_archive = pull_data->is_mirror && self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2;
if (error)
pull_data->async_error = &pull_data->cached_async_error;
else
@@ -2845,7 +2848,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
&from_revision, error))
goto out;
if (!disable_static_deltas && !pull_data->is_mirror &&
if (!disable_static_deltas && !mirroring_into_archive &&
(from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0))
{
if (!request_static_delta_superblock_sync (pull_data, from_revision, to_revision,
src/libostree/ostree-repo-static-delta-core.c
+10 -4
View File
@@ -795,7 +795,9 @@ _ostree_repo_static_delta_delete (OstreeRepo *self,
g_autofree char *deltadir = NULL;
struct stat buf;
_ostree_parse_delta_name (delta_id, &from, &to);
if (!_ostree_parse_delta_name (delta_id, &from, &to, error))
goto out;
deltadir = _ostree_get_relative_static_delta_path (from, to, NULL);
if (fstatat (self->repo_dir_fd, deltadir, &buf, 0) != 0)
@@ -830,7 +832,9 @@ _ostree_repo_static_delta_query_exists (OstreeRepo *self,
g_autofree char *superblock_path = NULL;
struct stat stbuf;
_ostree_parse_delta_name (delta_id, &from, &to);
if (!_ostree_parse_delta_name (delta_id, &from, &to, error))
return FALSE;
superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
if (fstatat (self->repo_dir_fd, superblock_path, &stbuf, 0) < 0)
@@ -854,7 +858,7 @@ gboolean
_ostree_repo_static_delta_dump (OstreeRepo *self,
const char *delta_id,
GCancellable *cancellable,
GError **error)
GError **error)
{
gboolean ret = FALSE;
g_autofree char *from = NULL;
@@ -868,7 +872,9 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
OstreeDeltaEndianness endianness;
gboolean swap_endian = FALSE;
_ostree_parse_delta_name (delta_id, &from, &to);
if (!_ostree_parse_delta_name (delta_id, &from, &to, error))
goto out;
superblock_path = _ostree_get_relative_static_delta_superblock_path (from, to);
if (!ot_util_variant_map_at (self->repo_dir_fd, superblock_path,
src/libostree/ostree-repo.c
+6 -3
View File
@@ -516,9 +516,9 @@ ostree_repo_finalize (GObject *object)
g_free (self->commit_stagedir_name);
glnx_release_lock_file (&self->commit_stagedir_lock);
g_clear_object (&self->tmp_dir);
if (self->tmp_dir_fd)
if (self->tmp_dir_fd != -1)
(void) close (self->tmp_dir_fd);
if (self->cache_dir_fd)
if (self->cache_dir_fd != -1)
(void) close (self->cache_dir_fd);
if (self->objects_dir_fd != -1)
(void) close (self->objects_dir_fd);
@@ -702,6 +702,7 @@ ostree_repo_init (OstreeRepo *self)
self->repo_dir_fd = -1;
self->cache_dir_fd = -1;
self->tmp_dir_fd = -1;
self->commit_stagedir_fd = -1;
self->objects_dir_fd = -1;
self->uncompressed_objects_dir_fd = -1;
@@ -4701,7 +4702,9 @@ ostree_repo_regenerate_summary (OstreeRepo *self,
glnx_fd_close int superblock_file_fd = -1;
g_autoptr(GInputStream) in_stream = NULL;
_ostree_parse_delta_name (delta_names->pdata[i], &from, &to);
if (!_ostree_parse_delta_name (delta_names->pdata[i], &from, &to, error))
goto out;
superblock = _ostree_get_relative_static_delta_superblock_path ((from && from[0]) ? from : NULL, to);
superblock_file_fd = openat (self->repo_dir_fd, superblock, O_RDONLY | O_CLOEXEC);
if (superblock_file_fd == -1)
src/libostree/ostree-sysroot-deploy.c
+63 -46
View File
@@ -894,6 +894,42 @@ merge_configuration (OstreeSysroot *sysroot,
return ret;
}
static gboolean
write_origin_file_internal (OstreeSysroot *sysroot,
OstreeDeployment *deployment,
GKeyFile *new_origin,
GLnxFileReplaceFlags flags,
GCancellable *cancellable,
GError **error)
{
GKeyFile *origin =
new_origin ? new_origin : ostree_deployment_get_origin (deployment);
if (origin)
{
g_autofree char *origin_path = NULL;
g_autofree char *contents = NULL;
gsize len;
origin_path = g_strdup_printf ("ostree/deploy/%s/deploy/%s.%d.origin",
ostree_deployment_get_osname (deployment),
ostree_deployment_get_csum (deployment),
ostree_deployment_get_deployserial (deployment));
contents = g_key_file_to_data (origin, &len, error);
if (!contents)
return FALSE;
if (!glnx_file_replace_contents_at (sysroot->sysroot_fd,
origin_path, (guint8*)contents, len,
flags,
cancellable, error))
return FALSE;
}
return TRUE;
}
/**
* ostree_sysroot_write_origin_file:
* @sysroot: System root
@@ -913,35 +949,9 @@ ostree_sysroot_write_origin_file (OstreeSysroot *sysroot,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
GKeyFile *origin =
new_origin ? new_origin : ostree_deployment_get_origin (deployment);
if (origin)
{
g_autoptr(GFile) deployment_path = ostree_sysroot_get_deployment_directory (sysroot, deployment);
g_autoptr(GFile) origin_path = ostree_sysroot_get_deployment_origin_path (deployment_path);
g_autoptr(GFile) origin_parent = g_file_get_parent (origin_path);
g_autofree char *contents = NULL;
gsize len;
g_autoptr(GBytes) contents_bytes = NULL;
contents = g_key_file_to_data (origin, &len, error);
if (!contents)
goto out;
contents_bytes = g_bytes_new_static (contents, len);
if (!ot_gfile_replace_contents_fsync (origin_path, contents_bytes,
cancellable, error))
goto out;
if (!ot_util_fsync_directory (origin_parent, cancellable, error))
goto out;
}
ret = TRUE;
out:
return ret;
return write_origin_file_internal (sysroot, deployment, new_origin,
GLNX_FILE_REPLACE_DATASYNC_NEW,
cancellable, error);
}
static gboolean
@@ -1635,25 +1645,28 @@ cleanup_legacy_current_symlinks (OstreeSysroot *self,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
guint i;
g_autoptr(GHashTable) created_current_for_osname =
g_hash_table_new (g_str_hash, g_str_equal);
g_autoptr(GString) buf = g_string_new ("");
for (i = 0; i < self->deployments->len; i++)
{
OstreeDeployment *deployment = self->deployments->pdata[i];
const char *osname = ostree_deployment_get_osname (deployment);
g_autoptr(GFile) osdir = ot_gfile_resolve_path_printf (self->path, "ostree/deploy/%s", osname);
g_autoptr(GFile) legacy_link = g_file_get_child (osdir, "current");
if (!ot_gfile_ensure_unlinked (legacy_link, cancellable, error))
goto out;
g_string_truncate (buf, 0);
g_string_append_printf (buf, "ostree/deploy/%s/current", osname);
if (unlinkat (self->sysroot_fd, buf->str, 0) < 0)
{
if (errno != ENOENT)
{
glnx_set_error_from_errno (error);
return FALSE;
}
}
}
ret = TRUE;
out:
return ret;
return TRUE;
}
static gboolean
@@ -1814,7 +1827,7 @@ _ostree_sysroot_write_deployments_internal (OstreeSysroot *self,
{
int new_bootversion = self->bootversion ? 0 : 1;
glnx_unref_object OstreeBootloader *bootloader = NULL;
g_autoptr(GFile) new_loader_entries_dir = NULL;
g_autofree char* new_loader_entries_dir = NULL;
glnx_unref_object OstreeRepo *repo = NULL;
gboolean show_osname = FALSE;
@@ -1835,11 +1848,11 @@ _ostree_sysroot_write_deployments_internal (OstreeSysroot *self,
if (!_ostree_sysroot_query_bootloader (self, &bootloader, cancellable, error))
goto out;
new_loader_entries_dir = ot_gfile_resolve_path_printf (self->path, "boot/loader.%d/entries",
new_bootversion);
if (!glnx_shutil_rm_rf_at (AT_FDCWD, gs_file_get_path_cached (new_loader_entries_dir), cancellable, error))
new_loader_entries_dir = g_strdup_printf ("boot/loader.%d/entries", new_bootversion);
if (!glnx_shutil_rm_rf_at (self->sysroot_fd, new_loader_entries_dir, cancellable, error))
goto out;
if (!ot_util_ensure_directory_and_fsync (new_loader_entries_dir, cancellable, error))
if (!glnx_shutil_mkdir_p_at (self->sysroot_fd, new_loader_entries_dir, 0755,
cancellable, error))
goto out;
/* Need the repo to try and extract the versions for deployments.
@@ -2152,8 +2165,12 @@ ostree_sysroot_deploy_tree (OstreeSysroot *self,
goto out;
}
if (!ostree_sysroot_write_origin_file (self, new_deployment, NULL,
cancellable, error))
/* Don't fsync here, as we assume that's all done in
* ostree_sysroot_write_deployments().
*/
if (!write_origin_file_internal (self, new_deployment, NULL,
GLNX_FILE_REPLACE_NODATASYNC,
cancellable, error))
{
g_prefix_error (error, "Writing out origin file: ");
goto out;
src/libostree/ostree-sysroot.c
+16 -1
View File
@@ -21,6 +21,8 @@
#include "config.h"
#include "otutil.h"
#include <sys/mount.h>
#include <sys/wait.h>
#include "ostree-core-private.h"
#include "ostree-sysroot-private.h"
@@ -260,7 +262,10 @@ void
ostree_sysroot_unload (OstreeSysroot *self)
{
if (self->sysroot_fd != -1)
(void) close (self->sysroot_fd);
{
(void) close (self->sysroot_fd);
self->sysroot_fd = -1;
}
}
/**
@@ -1484,6 +1489,16 @@ ostree_sysroot_init_osname (OstreeSysroot *self,
goto out;
}
/* This needs to be available and properly labeled early during the boot
* process (before tmpfiles.d kicks in), so that journald can flush logs from
* the first boot there. https://bugzilla.redhat.com/show_bug.cgi?id=1265295
* */
if (mkdirat (dfd, "var/log", 0755) < 0)
{
glnx_set_prefix_error_from_errno (error, "Creating %s", "var/log");
goto out;
}
if (symlinkat ("../run", dfd, "var/run") < 0)
{
glnx_set_prefix_error_from_errno (error, "Symlinking %s", "var/run");