packaging/ostree
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New upstream version 2017.3

Browse Source
This commit is contained in:
Simon McVittie committed 2017-03-10 23:16:34 +00:00
2 parents 6ee013cea0 e02e900206
commit 057de00126
134 files changed
+7527 -2384
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/libostree/bupsplit.c
+2 -36
View File
@@ -80,7 +80,8 @@ static uint32_t rollsum_digest(Rollsum *r)
}
static uint32_t rollsum_sum(uint8_t *buf, size_t ofs, size_t len)
uint32_t
bupsplit_sum(uint8_t *buf, size_t ofs, size_t len)
{
size_t count;
Rollsum r;
@@ -115,38 +116,3 @@ int bupsplit_find_ofs(const unsigned char *buf, int len, int *bits)
}
return 0;
}
#ifndef BUP_NO_SELFTEST
#define BUP_SELFTEST_SIZE 100000
int bupsplit_selftest()
{
uint8_t *buf = malloc(BUP_SELFTEST_SIZE);
uint32_t sum1a, sum1b, sum2a, sum2b, sum3a, sum3b;
unsigned count;
srandom(1);
for (count = 0; count < BUP_SELFTEST_SIZE; count++)
buf[count] = random();
sum1a = rollsum_sum(buf, 0, BUP_SELFTEST_SIZE);
sum1b = rollsum_sum(buf, 1, BUP_SELFTEST_SIZE);
sum2a = rollsum_sum(buf, BUP_SELFTEST_SIZE - BUP_WINDOWSIZE*5/2,
BUP_SELFTEST_SIZE - BUP_WINDOWSIZE);
sum2b = rollsum_sum(buf, 0, BUP_SELFTEST_SIZE - BUP_WINDOWSIZE);
sum3a = rollsum_sum(buf, 0, BUP_WINDOWSIZE+3);
sum3b = rollsum_sum(buf, 3, BUP_WINDOWSIZE+3);
fprintf(stderr, "sum1a = 0x%08x\n", sum1a);
fprintf(stderr, "sum1b = 0x%08x\n", sum1b);
fprintf(stderr, "sum2a = 0x%08x\n", sum2a);
fprintf(stderr, "sum2b = 0x%08x\n", sum2b);
fprintf(stderr, "sum3a = 0x%08x\n", sum3a);
fprintf(stderr, "sum3b = 0x%08x\n", sum3b);
free(buf);
return sum1a!=sum1b || sum2a!=sum2b || sum3a!=sum3b;
}
#endif // !BUP_NO_SELFTEST
src/libostree/bupsplit.h
+5 -2
View File
@@ -30,6 +30,9 @@
#ifndef __BUPSPLIT_H
#define __BUPSPLIT_H
#include <stdint.h>
#include <sys/types.h>
#define BUP_BLOBBITS (13)
#define BUP_BLOBSIZE (1<<BUP_BLOBBITS)
#define BUP_WINDOWBITS (7)
@@ -38,9 +41,9 @@
#ifdef __cplusplus
extern "C" {
#endif
uint32_t bupsplit_sum(uint8_t *buf, size_t ofs, size_t len);
int bupsplit_find_ofs(const unsigned char *buf, int len, int *bits);
int bupsplit_selftest(void);
#ifdef __cplusplus
}
src/libostree/libostree.sym
+28 -10
View File
@@ -366,21 +366,39 @@ global:
/* No new symbols in 2016.15 */
/* NOTE NOTE NOTE
* Versions above here are released. Only add symbols below this line.
* NOTE NOTE NOTE
*/
LIBOSTREE_2017.1 {
global:
ostree_repo_prune_from_reachable;
} LIBOSTREE_2016.14;
/* Stub section for the stable release *after* this development one; don't
* edit this other than to update the last number. This is just a copy/paste
* source.
LIBOSTREE_2016.XX {
LIBOSTREE_2017.2 {
global:
ostree_repo_reload_config;
} LIBOSTREE_2017.1;
LIBOSTREE_2017.3 {
global:
ostree_raw_file_to_archive_z2_stream_with_options;
} LIBOSTREE_2017.2;
/* NOTE NOTE NOTE
* Versions above here are released. Only add symbols below this line.
* NOTE NOTE NOTE
*/
/* Stub section for new version, uncomment when the first symbol is added
LIBOSTREE_2017.$NEWVERSION {
global:
someostree_symbol_deleteme;
} LIBOSTREE_2016.14;
} LIBOSTREE_2017.2;
*/
/* Stub section for the stable release *after* this development one; don't
* edit this other than to update the last number. This is just a copy/paste
* source. Replace $LASTSTABLE with the last stable version, and $NEWVERSION
* with whatever the next version with new symbols will be.
LIBOSTREE_2017.$NEWVERSION {
global:
someostree_symbol_deleteme;
} LIBOSTREE_2017.$LASTSTABLE;
*/
src/libostree/ostree-bootloader-grub2.c
+65 -29
View File
@@ -29,6 +29,20 @@
#include <string.h>
/* I only did some cursory research here, but it appears
* that we only want to use "linux16" for x86 platforms.
* At least, I got a report that "linux16" is definitely wrong
* for ppc64. See
* http://pkgs.fedoraproject.org/cgit/rpms/grub2.git/tree/0036-Use-linux16-when-appropriate-880840.patch?h=f25
* https://bugzilla.redhat.com/show_bug.cgi?id=1108296
* among others.
*/
#if defined(__i386__) || defined(__x86_64__)
#define GRUB2_USES_16 1
#else
#define GRUB2_USES_16 0
#endif
struct _OstreeBootloaderGrub2
{
GObject parent_instance;
@@ -203,7 +217,13 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot
if (is_efi)
g_string_append (output, "linuxefi ");
else
g_string_append (output, "linux16 ");
{
#if GRUB2_USES_16
g_string_append (output, "linux16 ");
#else
g_string_append (output, "linux ");
#endif
}
g_string_append (output, kernel);
options = ostree_bootconfig_parser_get (config, "options");
@@ -220,7 +240,13 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot
if (is_efi)
g_string_append (output, "initrdefi ");
else
g_string_append (output, "initrd16 ");
{
#if GRUB2_USES_16
g_string_append (output, "initrd16 ");
#else
g_string_append (output, "initrd ");
#endif
}
g_string_append (output, initrd);
g_string_append_c (output, '\n');
}
@@ -239,12 +265,26 @@ _ostree_bootloader_grub2_generate_config (OstreeSysroot *sysroot
return ret;
}
typedef struct {
const char *root;
const char *bootversion_str;
gboolean is_efi;
} Grub2ChildSetupData;
static void
grub2_child_setup (gpointer user_data)
{
const char *root = user_data;
Grub2ChildSetupData *cdata = user_data;
if (chdir (root) != 0)
setenv ("_OSTREE_GRUB2_BOOTVERSION", cdata->bootversion_str, TRUE);
/* We have to pass our state to the child */
if (cdata->is_efi)
setenv ("_OSTREE_GRUB2_IS_EFI", "1", TRUE);
if (!cdata->root)
return;
if (chdir (cdata->root) != 0)
{
perror ("chdir");
_exit (1);
@@ -268,7 +308,7 @@ grub2_child_setup (gpointer user_data)
_exit (1);
}
if (mount (root, "/", NULL, MS_MOVE, NULL) < 0)
if (mount (cdata->root, "/", NULL, MS_MOVE, NULL) < 0)
{
perror ("failed to MS_MOVE to /");
_exit (1);
@@ -290,14 +330,15 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader,
OstreeBootloaderGrub2 *self = OSTREE_BOOTLOADER_GRUB2 (bootloader);
gboolean ret = FALSE;
g_autoptr(GFile) new_config_path = NULL;
GSubprocessFlags subp_flags = 0;
glnx_unref_object GSubprocessLauncher *launcher = NULL;
glnx_unref_object GSubprocess *proc = NULL;
g_autofree char *bootversion_str = g_strdup_printf ("%u", (guint)bootversion);
g_autoptr(GFile) config_path_efi_dir = NULL;
g_autofree char *grub2_mkconfig_chroot = NULL;
gboolean use_system_grub2_mkconfig = TRUE;
const gchar *grub_exec = NULL;
const char *grub_argv[4] = { NULL, "-o", NULL, NULL};
GSpawnFlags grub_spawnflags = G_SPAWN_SEARCH_PATH;
int grub2_estatus;
Grub2ChildSetupData cdata = { NULL, };
#ifdef USE_BUILTIN_GRUB2_MKCONFIG
use_system_grub2_mkconfig = FALSE;
@@ -354,22 +395,15 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader,
bootversion);
}
if (!g_getenv ("OSTREE_DEBUG_GRUB2"))
subp_flags |= (G_SUBPROCESS_FLAGS_STDOUT_SILENCE | G_SUBPROCESS_FLAGS_STDERR_SILENCE);
launcher = g_subprocess_launcher_new (subp_flags);
g_subprocess_launcher_setenv (launcher, "_OSTREE_GRUB2_BOOTVERSION", bootversion_str, TRUE);
/* We have to pass our state to the child */
if (self->is_efi)
g_subprocess_launcher_setenv (launcher, "_OSTREE_GRUB2_IS_EFI", "1", TRUE);
/* We need to chroot() if we're not in /. This assumes our caller has
* set up the bind mounts outside.
*/
if (grub2_mkconfig_chroot != NULL)
g_subprocess_launcher_set_child_setup (launcher, grub2_child_setup, grub2_mkconfig_chroot, NULL);
grub_argv[0] = grub_exec;
grub_argv[2] = gs_file_get_path_cached (new_config_path);
/* In the current Fedora grub2 package, this script doesn't even try
if (!g_getenv ("OSTREE_DEBUG_GRUB2"))
grub_spawnflags |= G_SPAWN_STDOUT_TO_DEV_NULL | G_SPAWN_STDERR_TO_DEV_NULL;
cdata.root = grub2_mkconfig_chroot;
cdata.bootversion_str = bootversion_str;
cdata.is_efi = self->is_efi;
/* Note in older versions of the grub2 package, this script doesn't even try
to be atomic; it just does:
cat ${grub_cfg}.new > ${grub_cfg}
@@ -377,13 +411,15 @@ _ostree_bootloader_grub2_write_config (OstreeBootloader *bootloader,
Upstream is fixed though.
*/
proc = g_subprocess_launcher_spawn (launcher, error,
grub_exec, "-o",
gs_file_get_path_cached (new_config_path),
NULL);
if (!g_subprocess_wait_check (proc, cancellable, error))
if (!g_spawn_sync (NULL, (char**)grub_argv, NULL, grub_spawnflags,
grub2_child_setup, &cdata, NULL, NULL,
&grub2_estatus, error))
goto out;
if (!g_spawn_check_exit_status (grub2_estatus, error))
{
g_prefix_error (error, "%s: ", grub_argv[0]);
goto out;
}
/* Now let's fdatasync() for the new file */
{ glnx_fd_close int new_config_fd = open (gs_file_get_path_cached (new_config_path), O_RDONLY | O_CLOEXEC);
src/libostree/ostree-core-private.h
+15
View File
@@ -24,6 +24,9 @@
G_BEGIN_DECLS
/* It's what gzip does, 9 is too slow */
#define OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL (6)
/* This file contains private implementation data format definitions
* read by multiple implementation .c files.
*/
@@ -143,4 +146,16 @@ _ostree_detached_metadata_append_gpg_sig (GVariant *existing_metadata,
GFile *
_ostree_get_default_sysroot_path (void);
_OSTREE_PUBLIC
gboolean
_ostree_raw_file_to_archive_stream (GInputStream *input,
GFileInfo *file_info,
GVariant *xattrs,
guint compression_level,
GInputStream **out_input,
GCancellable *cancellable,
GError **error);
G_END_DECLS
src/libostree/ostree-core.c
+69 -15
View File
@@ -453,6 +453,34 @@ header_and_input_to_stream (GVariant *file_header,
return TRUE;
}
gboolean
_ostree_raw_file_to_archive_stream (GInputStream *input,
GFileInfo *file_info,
GVariant *xattrs,
guint compression_level,
GInputStream **out_input,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GVariant) file_header = NULL;
g_autoptr(GInputStream) zlib_input = NULL;
file_header = _ostree_zlib_file_header_new (file_info, xattrs);
if (input != NULL)
{
g_autoptr(GConverter) zlib_compressor = NULL;
zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, compression_level));
zlib_input = g_converter_input_stream_new (input, zlib_compressor);
}
return header_and_input_to_stream (file_header,
zlib_input,
out_input,
NULL,
cancellable,
error);
}
/**
* ostree_raw_file_to_archive_z2_stream:
* @input: File raw content stream
@@ -473,23 +501,49 @@ ostree_raw_file_to_archive_z2_stream (GInputStream *input,
GCancellable *cancellable,
GError **error)
{
g_autoptr(GVariant) file_header = NULL;
g_autoptr(GInputStream) zlib_input = NULL;
return _ostree_raw_file_to_archive_stream (input, file_info, xattrs,
OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL,
out_input, cancellable, error);
}
file_header = _ostree_zlib_file_header_new (file_info, xattrs);
if (input != NULL)
{
g_autoptr(GConverter) zlib_compressor = NULL;
/**
* ostree_raw_file_to_archive_z2_stream_with_options:
* @input: File raw content stream
* @file_info: A file info
* @xattrs: (allow-none): Optional extended attributes
* @options: (nullable): A GVariant `a{sv}` with an extensible set of flags
* @out_input: (out): Serialized object stream
* @cancellable: Cancellable
* @error: Error
*
* Like ostree_raw_file_to_archive_z2_stream(), but supports an extensible set
* of flags. The following flags are currently defined:
*
* - `compression-level` (`i`): Level of compression to use, 0–9, with 0 being
* the least compression, and <0 giving the default level (currently 6).
*
* Since: 2017.3
*/
gboolean
ostree_raw_file_to_archive_z2_stream_with_options (GInputStream *input,
GFileInfo *file_info,
GVariant *xattrs,
GVariant *options,
GInputStream **out_input,
GCancellable *cancellable,
GError **error)
{
gint compression_level = -1;
zlib_compressor = G_CONVERTER (g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9));
zlib_input = g_converter_input_stream_new (input, zlib_compressor);
}
return header_and_input_to_stream (file_header,
zlib_input,
out_input,
NULL,
cancellable,
error);
if (options)
(void) g_variant_lookup (options, "compression-level", "i", &compression_level);
if (compression_level < 0)
compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL;
return _ostree_raw_file_to_archive_stream (input, file_info, xattrs,
compression_level,
out_input, cancellable, error);
}
/**
src/libostree/ostree-core.h
+10
View File
@@ -311,6 +311,16 @@ ostree_raw_file_to_archive_z2_stream (GInputStream *input,
GCancellable *cancellable,
GError **error);
_OSTREE_PUBLIC
gboolean
ostree_raw_file_to_archive_z2_stream_with_options (GInputStream *input,
GFileInfo *file_info,
GVariant *xattrs,
GVariant *options,
GInputStream **out_input,
GCancellable *cancellable,
GError **error);
_OSTREE_PUBLIC
gboolean ostree_raw_file_to_content_stream (GInputStream *input,
GFileInfo *file_info,
src/libostree/ostree-enumtypes.c
-86
View File
@@ -1,86 +0,0 @@
/* Generated data (by glib-mkenums) */
/*
* Copyright (C) 2015 Red Hat, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include <string.h>
#include "ostree-enumtypes.h"
/* enumerations from "./src/libostree/ostree-fetcher.h" */
#include "./src/libostree/ostree-fetcher.h"
GType
_ostree_fetcher_config_flags_get_type (void)
{
static volatile gsize the_type__volatile = 0;
if (g_once_init_enter (&the_type__volatile))
{
static const GFlagsValue values[] = {
{ OSTREE_FETCHER_FLAGS_NONE,
"OSTREE_FETCHER_FLAGS_NONE",
"none" },
{ OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE,
"OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE",
"tls-permissive" },
{ 0, NULL, NULL }
};
GType the_type = g_flags_register_static (
g_intern_static_string ("OstreeFetcherConfigFlags"),
values);
g_once_init_leave (&the_type__volatile, the_type);
}
return the_type__volatile;
}
GType
_ostree_fetcher_request_flags_get_type (void)
{
static volatile gsize the_type__volatile = 0;
if (g_once_init_enter (&the_type__volatile))
{
static const GFlagsValue values[] = {
{ OSTREE_FETCHER_REQUEST_NUL_TERMINATION,
"OSTREE_FETCHER_REQUEST_NUL_TERMINATION",
"termination" },
{ 0, NULL, NULL }
};
GType the_type = g_flags_register_static (
g_intern_static_string ("OstreeFetcherRequestFlags"),
values);
g_once_init_leave (&the_type__volatile, the_type);
}
return the_type__volatile;
}
/* Generated data ends here */
src/libostree/ostree-enumtypes.h
-41
View File
@@ -1,41 +0,0 @@
/* Generated data (by glib-mkenums) */
/*
* Copyright (C) 2015 Red Hat, Inc.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#pragma once
#include <glib-object.h>
G_BEGIN_DECLS
/* Enumerations from "./src/libostree/ostree-fetcher.h" */
#define OSTREE_TYPE_FETCHER_CONFIG_FLAGS (_ostree_fetcher_config_flags_get_type ())
GType _ostree_fetcher_config_flags_get_type (void) G_GNUC_CONST;
#define OSTREE_TYPE_FETCHER_REQUEST_FLAGS (_ostree_fetcher_request_flags_get_type ())
GType _ostree_fetcher_request_flags_get_type (void) G_GNUC_CONST;
G_END_DECLS
/* Generated data ends here */
src/libostree/ostree-fetcher-curl.c
+941
View File
@@ -0,0 +1,941 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2016 Colin Walters <walters@verbum.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*/
#include "config.h"
#include <gio/gfiledescriptorbased.h>
#include <gio/gunixoutputstream.h>
#include <glib-unix.h>
#include <curl/curl.h>
/* These macros came from 7.43.0, but we want to check
* for versions a bit earlier than that (to work on CentOS 7),
* so define them here if we're using an older version.
*/
#ifndef CURL_VERSION_BITS
#define CURL_VERSION_BITS(x,y,z) ((x)<<16|(y)<<8|z)
#endif
#ifndef CURL_AT_LEAST_VERSION
#define CURL_AT_LEAST_VERSION(x,y,z) (LIBCURL_VERSION_NUM >= CURL_VERSION_BITS(x, y, z))
#endif
#include "ostree-fetcher.h"
#include "ostree-fetcher-util.h"
#include "ostree-enumtypes.h"
#include "ostree-repo-private.h"
#include "otutil.h"
#include "ostree-soup-uri.h"
typedef struct FetcherRequest FetcherRequest;
typedef struct SockInfo SockInfo;
static int sock_cb (CURL *e, curl_socket_t s, int what, void *cbp, void *sockp);
static gboolean timer_cb (gpointer data);
static void sock_unref (SockInfo *f);
static int update_timeout_cb (CURLM *multi, long timeout_ms, void *userp);
static void request_unref (FetcherRequest *req);
static void initiate_next_curl_request (FetcherRequest *req, GTask *task);
static void destroy_and_unref_source (GSource *source);
struct OstreeFetcher
{
GObject parent_instance;
OstreeFetcherConfigFlags config_flags;
char *remote_name;
char *tls_ca_db_path;
char *tls_client_cert_path;
char *tls_client_key_path;
char *cookie_jar_path;
char *proxy;
struct curl_slist *extra_headers;
int tmpdir_dfd;
GMainContext *mainctx;
CURLM *multi;
GSource *timer_event;
int curl_running;
GHashTable *outstanding_requests; /* Set<GTask> */
GHashTable *sockets; /* Set<SockInfo> */
guint64 bytes_transferred;
};
/* Information associated with a request */
struct FetcherRequest {
guint refcount;
GPtrArray *mirrorlist;
guint idx;
char *filename;
guint64 current_size;
guint64 max_size;
OstreeFetcherRequestFlags flags;
gboolean is_membuf;
GError *caught_write_error;
char *out_tmpfile;
int out_tmpfile_fd;
GString *output_buf;
CURL *easy;
char error[CURL_ERROR_SIZE];
OstreeFetcher *fetcher;
};
/* Information associated with a specific socket */
struct SockInfo {
guint refcount;
curl_socket_t sockfd;
int action;
long timeout;
GSource *ch;
OstreeFetcher *fetcher;
};
enum {
PROP_0,
PROP_CONFIG_FLAGS
};
G_DEFINE_TYPE (OstreeFetcher, _ostree_fetcher, G_TYPE_OBJECT)
static void
_ostree_fetcher_set_property (GObject *object,
guint prop_id,
const GValue *value,
GParamSpec *pspec)
{
OstreeFetcher *self = OSTREE_FETCHER (object);
switch (prop_id)
{
case PROP_CONFIG_FLAGS:
self->config_flags = g_value_get_flags (value);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
_ostree_fetcher_get_property (GObject *object,
guint prop_id,
GValue *value,
GParamSpec *pspec)
{
OstreeFetcher *self = OSTREE_FETCHER (object);
switch (prop_id)
{
case PROP_CONFIG_FLAGS:
g_value_set_flags (value, self->config_flags);
break;
default:
G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
break;
}
}
static void
_ostree_fetcher_finalize (GObject *object)
{
OstreeFetcher *self = OSTREE_FETCHER (object);
g_free (self->remote_name);
g_free (self->cookie_jar_path);
g_free (self->proxy);
g_assert_cmpint (g_hash_table_size (self->outstanding_requests), ==, 0);
g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all);
g_hash_table_unref (self->outstanding_requests);
g_hash_table_unref (self->sockets);
g_clear_pointer (&self->timer_event, (GDestroyNotify)destroy_and_unref_source);
if (self->mainctx)
g_main_context_unref (self->mainctx);
curl_multi_cleanup (self->multi);
G_OBJECT_CLASS (_ostree_fetcher_parent_class)->finalize (object);
}
static void
_ostree_fetcher_constructed (GObject *object)
{
// OstreeFetcher *self = OSTREE_FETCHER (object);
G_OBJECT_CLASS (_ostree_fetcher_parent_class)->constructed (object);
}
static void
_ostree_fetcher_class_init (OstreeFetcherClass *klass)
{
GObjectClass *gobject_class = G_OBJECT_CLASS (klass);
gobject_class->set_property = _ostree_fetcher_set_property;
gobject_class->get_property = _ostree_fetcher_get_property;
gobject_class->finalize = _ostree_fetcher_finalize;
gobject_class->constructed = _ostree_fetcher_constructed;
g_object_class_install_property (gobject_class,
PROP_CONFIG_FLAGS,
g_param_spec_flags ("config-flags",
"",
"",
OSTREE_TYPE_FETCHER_CONFIG_FLAGS,
OSTREE_FETCHER_FLAGS_NONE,
G_PARAM_READWRITE |
G_PARAM_CONSTRUCT_ONLY |
G_PARAM_STATIC_STRINGS));
}
static void
_ostree_fetcher_init (OstreeFetcher *self)
{
self->multi = curl_multi_init();
self->outstanding_requests = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)g_object_unref, NULL);
self->sockets = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)sock_unref, NULL);
curl_multi_setopt (self->multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
curl_multi_setopt (self->multi, CURLMOPT_SOCKETDATA, self);
curl_multi_setopt (self->multi, CURLMOPT_TIMERFUNCTION, update_timeout_cb);
curl_multi_setopt (self->multi, CURLMOPT_TIMERDATA, self);
#if CURL_AT_LEAST_VERSION(7, 30, 0)
/* Let's do something reasonable here. */
curl_multi_setopt (self->multi, CURLMOPT_MAX_TOTAL_CONNECTIONS, 8);
#endif
}
OstreeFetcher *
_ostree_fetcher_new (int tmpdir_dfd,
const char *remote_name,
OstreeFetcherConfigFlags flags)
{
OstreeFetcher *fetcher = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL);
fetcher->remote_name = g_strdup (remote_name);
fetcher->tmpdir_dfd = tmpdir_dfd;
return fetcher;
}
static void
destroy_and_unref_source (GSource *source)
{
g_source_destroy (source);
g_source_unref (source);
}
static char *
request_get_uri (FetcherRequest *req, guint idx)
{
SoupURI *baseuri = req->mirrorlist->pdata[idx];
if (!req->filename)
return soup_uri_to_string (baseuri, FALSE);
{ g_autofree char *uristr = soup_uri_to_string (baseuri, FALSE);
return g_build_filename (uristr, req->filename, NULL);
}
}
static gboolean
ensure_tmpfile (FetcherRequest *req, GError **error)
{
if (req->out_tmpfile_fd == -1)
{
if (!glnx_open_tmpfile_linkable_at (req->fetcher->tmpdir_dfd, ".",
O_WRONLY, &req->out_tmpfile_fd,
&req->out_tmpfile,
error))
return FALSE;
}
return TRUE;
}
/* Check for completed transfers, and remove their easy handles */
static void
check_multi_info (OstreeFetcher *fetcher)
{
CURLMsg *msg;
int msgs_left;
while ((msg = curl_multi_info_read (fetcher->multi, &msgs_left)) != NULL)
{
long response;
CURL *easy = msg->easy_handle;
CURLcode curlres = msg->data.result;
GTask *task;
FetcherRequest *req;
const char *eff_url;
gboolean is_file;
gboolean continued_request = FALSE;
if (msg->msg != CURLMSG_DONE)
continue;
curl_easy_getinfo (easy, CURLINFO_PRIVATE, &task);
curl_easy_getinfo (easy, CURLINFO_EFFECTIVE_URL, &eff_url);
/* We should have limited the protocols; this is what
* curl's tool_operate.c does.
*/
is_file = g_str_has_prefix (eff_url, "file:");
g_assert (is_file || g_str_has_prefix (eff_url, "http"));
req = g_task_get_task_data (task);
if (req->caught_write_error)
g_task_return_error (task, g_steal_pointer (&req->caught_write_error));
else if (curlres != CURLE_OK)
{
if (is_file && curlres == CURLE_FILE_COULDNT_READ_FILE)
{
/* Handle file not found */
g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"%s",
curl_easy_strerror (curlres));
}
else
{
g_task_return_new_error (task, G_IO_ERROR, G_IO_ERROR_FAILED, "[%u] %s",
curlres,
curl_easy_strerror (curlres));
if (req->fetcher->remote_name)
_ostree_fetcher_journal_failure (req->fetcher->remote_name,
eff_url, curl_easy_strerror (curlres));
}
}
else
{
curl_easy_getinfo (easy, CURLINFO_RESPONSE_CODE, &response);
if (!is_file && !(response >= 200 && response < 300))
{
GIOErrorEnum giocode;
/* TODO - share with soup */
switch (response)
{
case 404:
case 403:
case 410:
giocode = G_IO_ERROR_NOT_FOUND;
break;
default:
giocode = G_IO_ERROR_FAILED;
}
if (req->idx + 1 == req->mirrorlist->len)
{
g_autofree char *msg = g_strdup_printf ("Server returned HTTP %lu", response);
g_task_return_new_error (task, G_IO_ERROR, giocode,
"%s", msg);
if (req->fetcher->remote_name)
_ostree_fetcher_journal_failure (req->fetcher->remote_name,
eff_url, msg);
}
else
{
continued_request = TRUE;
}
}
else if (req->is_membuf)
{
GBytes *ret;
if ((req->flags & OSTREE_FETCHER_REQUEST_NUL_TERMINATION) > 0)
g_string_append_c (req->output_buf, '\0');
ret = g_string_free_to_bytes (req->output_buf);
req->output_buf = NULL;
g_task_return_pointer (task, ret, (GDestroyNotify)g_bytes_unref);
}
else
{
g_autoptr(GError) local_error = NULL;
GError **error = &local_error;
/* TODO - share file naming with soup, and fix it */
g_autofree char *tmpfile_path =
g_compute_checksum_for_string (G_CHECKSUM_SHA256,
eff_url, strlen (eff_url));
if (!ensure_tmpfile (req, error))
{
g_task_return_error (task, g_steal_pointer (&local_error));
}
else if (fchmod (req->out_tmpfile_fd, 0644) < 0)
{
glnx_set_error_from_errno (error);
g_task_return_error (task, g_steal_pointer (&local_error));
}
else if (!glnx_link_tmpfile_at (fetcher->tmpdir_dfd,
GLNX_LINK_TMPFILE_REPLACE,
req->out_tmpfile_fd,
req->out_tmpfile,
fetcher->tmpdir_dfd,
tmpfile_path,
error))
g_task_return_error (task, g_steal_pointer (&local_error));
else
{
g_task_return_pointer (task, g_steal_pointer (&tmpfile_path), g_free);
}
}
}
curl_multi_remove_handle (fetcher->multi, easy);
if (continued_request)
{
req->idx++;
initiate_next_curl_request (req, task);
}
else
{
g_hash_table_remove (fetcher->outstanding_requests, task);
if (g_hash_table_size (fetcher->outstanding_requests) == 0)
{
g_clear_pointer (&fetcher->mainctx, g_main_context_unref);
}
}
}
}
/* Called by glib when our timeout expires */
static gboolean
timer_cb (gpointer data)
{
OstreeFetcher *fetcher = data;
CURLMcode rc;
GSource *orig_src = fetcher->timer_event;
rc = curl_multi_socket_action (fetcher->multi, CURL_SOCKET_TIMEOUT, 0, &fetcher->curl_running);
g_assert (rc == CURLM_OK);
check_multi_info (fetcher);
if (fetcher->timer_event == orig_src)
fetcher->timer_event = NULL;
return FALSE;
}
/* Update the event timer after curl_multi library calls */
static int
update_timeout_cb (CURLM *multi, long timeout_ms, void *userp)
{
OstreeFetcher *fetcher = userp;
g_clear_pointer (&fetcher->timer_event, (GDestroyNotify)destroy_and_unref_source);
if (timeout_ms != -1)
{
fetcher->timer_event = g_timeout_source_new (timeout_ms);
g_source_set_callback (fetcher->timer_event, timer_cb, fetcher, NULL);
g_source_attach (fetcher->timer_event, fetcher->mainctx);
}
return 0;
}
/* Called by glib when we get action on a multi socket */
static gboolean
event_cb (int fd, GIOCondition condition, gpointer data)
{
OstreeFetcher *fetcher = data;
CURLMcode rc;
int action =
(condition & G_IO_IN ? CURL_CSELECT_IN : 0) |
(condition & G_IO_OUT ? CURL_CSELECT_OUT : 0);
rc = curl_multi_socket_action (fetcher->multi, fd, action, &fetcher->curl_running);
g_assert (rc == CURLM_OK);
check_multi_info (fetcher);
if (fetcher->curl_running > 0)
{
return TRUE;
}
else
{
return FALSE;
}
}
/* Clean up the SockInfo structure */
static void
sock_unref (SockInfo *f)
{
if (!f)
return;
if (--f->refcount != 0)
return;
g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source);
g_free (f);
}
/* Assign information to a SockInfo structure */
static void
setsock (SockInfo*f, curl_socket_t s, int act, OstreeFetcher *fetcher)
{
GIOCondition kind =
(act&CURL_POLL_IN?G_IO_IN:0)|(act&CURL_POLL_OUT?G_IO_OUT:0);
f->sockfd = s;
f->action = act;
g_clear_pointer (&f->ch, (GDestroyNotify)destroy_and_unref_source);
/* TODO - investigate new g_source_modify_unix_fd() so changing the poll
* flags involves less allocation.
*/
f->ch = g_unix_fd_source_new (f->sockfd, kind);
g_source_set_callback (f->ch, (GSourceFunc) event_cb, fetcher, NULL);
g_source_attach (f->ch, fetcher->mainctx);
}
/* Initialize a new SockInfo structure */
static void
addsock (curl_socket_t s, CURL *easy, int action, OstreeFetcher *fetcher)
{
SockInfo *fdp = g_new0 (SockInfo, 1);
fdp->refcount = 1;
fdp->fetcher = fetcher;
setsock (fdp, s, action, fetcher);
curl_multi_assign (fetcher->multi, s, fdp);
g_hash_table_add (fetcher->sockets, fdp);
}
/* CURLMOPT_SOCKETFUNCTION */
static int
sock_cb (CURL *easy, curl_socket_t s, int what, void *cbp, void *sockp)
{
OstreeFetcher *fetcher = cbp;
SockInfo *fdp = (SockInfo*) sockp;
if (what == CURL_POLL_REMOVE)
{
if (!g_hash_table_remove (fetcher->sockets, fdp))
g_assert_not_reached ();
}
else
{
if (!fdp)
{
addsock (s, easy, what, fetcher);
}
else
{
setsock (fdp, s, what, fetcher);
}
}
return 0;
}
/* CURLOPT_WRITEFUNCTION */
static size_t
write_cb (void *ptr, size_t size, size_t nmemb, void *data)
{
const size_t realsize = size * nmemb;
GTask *task = data;
FetcherRequest *req;
req = g_task_get_task_data (task);
if (req->caught_write_error)
return -1;
if (req->max_size > 0)
{
if (realsize > req->max_size ||
(realsize + req->current_size) > req->max_size)
{
const char *eff_url;
curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url);
req->caught_write_error = g_error_new (G_IO_ERROR, G_IO_ERROR_FAILED,
"URI %s exceeded maximum size of %" G_GUINT64_FORMAT " bytes",
eff_url, req->max_size);
return -1;
}
}
if (req->is_membuf)
g_string_append_len (req->output_buf, ptr, realsize);
else
{
if (!ensure_tmpfile (req, &req->caught_write_error))
return -1;
g_assert (req->out_tmpfile_fd >= 0);
if (glnx_loop_write (req->out_tmpfile_fd, ptr, realsize) < 0)
{
glnx_set_error_from_errno (&req->caught_write_error);
return -1;
}
}
req->current_size += realsize;
req->fetcher->bytes_transferred += realsize;
return realsize;
}
/* CURLOPT_PROGRESSFUNCTION */
static int
prog_cb (void *p, double dltotal, double dlnow, double ult, double uln)
{
GTask *task = p;
FetcherRequest *req;
char *eff_url;
req = g_task_get_task_data (task);
curl_easy_getinfo (req->easy, CURLINFO_EFFECTIVE_URL, &eff_url);
g_printerr ("Progress: %s (%g/%g)\n", eff_url, dlnow, dltotal);
return 0;
}
static void
request_unref (FetcherRequest *req)
{
if (--req->refcount != 0)
return;
g_ptr_array_unref (req->mirrorlist);
g_free (req->filename);
g_clear_error (&req->caught_write_error);
if (req->out_tmpfile_fd != -1)
(void) close (req->out_tmpfile_fd);
g_free (req->out_tmpfile);
if (req->output_buf)
g_string_free (req->output_buf, TRUE);
curl_easy_cleanup (req->easy);
g_free (req);
}
int
_ostree_fetcher_get_dfd (OstreeFetcher *fetcher)
{
return fetcher->tmpdir_dfd;
}
void
_ostree_fetcher_set_proxy (OstreeFetcher *self,
const char *http_proxy)
{
g_free (self->proxy);
self->proxy = g_strdup (http_proxy);
}
void
_ostree_fetcher_set_cookie_jar (OstreeFetcher *self,
const char *jar_path)
{
g_free (self->cookie_jar_path);
self->cookie_jar_path = g_strdup (jar_path);
}
void
_ostree_fetcher_set_client_cert (OstreeFetcher *self,
const char *cert_path,
const char *key_path)
{
g_assert ((cert_path == NULL && key_path == NULL)
|| (cert_path != NULL && key_path != NULL));
g_free (self->tls_client_cert_path);
self->tls_client_cert_path = g_strdup (cert_path);
g_free (self->tls_client_key_path);
self->tls_client_key_path = g_strdup (key_path);
}
void
_ostree_fetcher_set_tls_database (OstreeFetcher *self,
const char *dbpath)
{
g_free (self->tls_ca_db_path);
self->tls_ca_db_path = g_strdup (dbpath);
}
void
_ostree_fetcher_set_extra_headers (OstreeFetcher *self,
GVariant *extra_headers)
{
GVariantIter viter;
const char *key;
const char *value;
g_clear_pointer (&self->extra_headers, (GDestroyNotify)curl_slist_free_all);
g_variant_iter_init (&viter, extra_headers);
while (g_variant_iter_loop (&viter, "(&s&s)", &key, &value))
{
g_autofree char *header = g_strdup_printf ("%s: %s", key, value);
self->extra_headers = curl_slist_append (self->extra_headers, header);
}
}
/* Re-bind all of the outstanding curl items to our new main context */
static void
adopt_steal_mainctx (OstreeFetcher *self,
GMainContext *mainctx)
{
GHashTableIter hiter;
gpointer key, value;
g_assert (self->mainctx == NULL);
self->mainctx = mainctx; /* Transfer */
if (self->timer_event != NULL)
{
guint64 readytime = g_source_get_ready_time (self->timer_event);
guint64 curtime = g_source_get_time (self->timer_event);
guint64 timeout_micros = curtime - readytime;
if (timeout_micros < 0)
timeout_micros = 0;
update_timeout_cb (self->multi, timeout_micros / 1000, self);
}
g_hash_table_iter_init (&hiter, self->sockets);
while (g_hash_table_iter_next (&hiter, &key, &value))
{
SockInfo *fdp = key;
setsock (fdp, fdp->sockfd, fdp->action, self);
}
}
static void
initiate_next_curl_request (FetcherRequest *req,
GTask *task)
{
CURLMcode rc;
OstreeFetcher *self = req->fetcher;
if (req->easy)
curl_easy_cleanup (req->easy);
req->easy = curl_easy_init ();
g_assert (req->easy);
g_assert_cmpint (req->idx, <, req->mirrorlist->len);
{ g_autofree char *uri = request_get_uri (req, req->idx);
curl_easy_setopt (req->easy, CURLOPT_URL, uri);
}
curl_easy_setopt (req->easy, CURLOPT_USERAGENT, "ostree ");
if (self->extra_headers)
curl_easy_setopt (req->easy, CURLOPT_HTTPHEADER, self->extra_headers);
if (self->cookie_jar_path)
{
rc = curl_easy_setopt (req->easy, CURLOPT_COOKIEFILE, self->cookie_jar_path);
g_assert_cmpint (rc, ==, CURLM_OK);
rc = curl_easy_setopt (req->easy, CURLOPT_COOKIELIST, "RELOAD");
g_assert_cmpint (rc, ==, CURLM_OK);
}
if (self->proxy)
{
rc = curl_easy_setopt (req->easy, CURLOPT_PROXY, self->proxy);
g_assert_cmpint (rc, ==, CURLM_OK);
}
if (self->tls_ca_db_path)
curl_easy_setopt (req->easy, CURLOPT_CAINFO, self->tls_ca_db_path);
if ((self->config_flags & OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE) > 0)
curl_easy_setopt (req->easy, CURLOPT_SSL_VERIFYPEER, 0L);
if (self->tls_client_cert_path)
{
curl_easy_setopt (req->easy, CURLOPT_SSLCERT, self->tls_client_cert_path);
curl_easy_setopt (req->easy, CURLOPT_SSLKEY, self->tls_client_key_path);
}
/* We should only speak HTTP; TODO: only enable file if specified */
curl_easy_setopt (req->easy, CURLOPT_PROTOCOLS, (long)(CURLPROTO_HTTP | CURLPROTO_HTTPS | CURLPROTO_FILE));
/* Picked the current version in F25 as of 20170127, since
* there are numerous HTTP/2 fixes since the original version in
* libcurl 7.43.0.
*/
#if CURL_AT_LEAST_VERSION(7, 51, 0)
curl_easy_setopt (req->easy, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
#endif
curl_easy_setopt (req->easy, CURLOPT_WRITEFUNCTION, write_cb);
if (g_getenv ("OSTREE_DEBUG_HTTP"))
curl_easy_setopt (req->easy, CURLOPT_VERBOSE, 1L);
curl_easy_setopt (req->easy, CURLOPT_ERRORBUFFER, req->error);
/* Note that the "easy" object's privdata is the task */
curl_easy_setopt (req->easy, CURLOPT_NOPROGRESS, 1L);
curl_easy_setopt (req->easy, CURLOPT_PROGRESSFUNCTION, prog_cb);
curl_easy_setopt (req->easy, CURLOPT_FOLLOWLOCATION, 1L);
curl_easy_setopt (req->easy, CURLOPT_CONNECTTIMEOUT, 30L);
curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_LIMIT, 1L);
curl_easy_setopt (req->easy, CURLOPT_LOW_SPEED_TIME, 30L);
/* closure bindings -> task */
curl_easy_setopt (req->easy, CURLOPT_PRIVATE, task);
curl_easy_setopt (req->easy, CURLOPT_WRITEDATA, task);
curl_easy_setopt (req->easy, CURLOPT_PROGRESSDATA, task);
rc = curl_multi_add_handle (self->multi, req->easy);
g_assert (rc == CURLM_OK);
}
static void
_ostree_fetcher_request_async (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
gboolean is_membuf,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
g_autoptr(GTask) task = NULL;
FetcherRequest *req;
g_autoptr(GMainContext) mainctx = g_main_context_ref_thread_default ();
/* We don't support multiple concurrent main contexts; take
* a ref to the first one, and require that later invocations
* share it.
*/
if (g_hash_table_size (self->outstanding_requests) == 0
&& mainctx != self->mainctx)
{
adopt_steal_mainctx (self, g_steal_pointer (&mainctx));
}
else
{
g_assert (self->mainctx == mainctx);
}
req = g_new0 (FetcherRequest, 1);
req->refcount = 1;
req->error[0]='\0';
req->fetcher = self;
req->mirrorlist = g_ptr_array_ref (mirrorlist);
req->filename = g_strdup (filename);
req->max_size = max_size;
req->flags = flags;
req->is_membuf = is_membuf;
/* We'll allocate the tmpfile on demand, so we handle
* file I/O errors just in the write func.
*/
req->out_tmpfile_fd = -1;
if (req->is_membuf)
req->output_buf = g_string_new ("");
task = g_task_new (self, cancellable, callback, user_data);
/* We'll use the GTask priority for our own priority queue. */
g_task_set_priority (task, priority);
g_task_set_source_tag (task, _ostree_fetcher_request_async);
g_task_set_task_data (task, req, (GDestroyNotify) request_unref);
initiate_next_curl_request (req, task);
g_hash_table_add (self->outstanding_requests, g_steal_pointer (&task));
/* Sanity check, I added * 2 just so we don't abort if something odd happens,
* but we do want to abort if we're asked to do obviously too many requests.
*/
g_assert_cmpint (g_hash_table_size (self->outstanding_requests), <,
_OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS * 2);
}
void
_ostree_fetcher_request_to_tmpfile (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
_ostree_fetcher_request_async (self, mirrorlist, filename, 0, FALSE,
max_size, priority, cancellable,
callback, user_data);
}
gboolean
_ostree_fetcher_request_to_tmpfile_finish (OstreeFetcher *self,
GAsyncResult *result,
char **out_filename,
GError **error)
{
GTask *task;
FetcherRequest *req;
gpointer ret;
g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
task = (GTask*)result;
req = g_task_get_task_data (task);
ret = g_task_propagate_pointer (task, error);
if (!ret)
return FALSE;
g_assert (!req->is_membuf);
g_assert (out_filename);
*out_filename = ret;
return TRUE;
}
void
_ostree_fetcher_request_to_membuf (OstreeFetcher *self,
GPtrArray *mirrorlist,
const char *filename,
OstreeFetcherRequestFlags flags,
guint64 max_size,
int priority,
GCancellable *cancellable,
GAsyncReadyCallback callback,
gpointer user_data)
{
_ostree_fetcher_request_async (self, mirrorlist, filename, flags, TRUE,
max_size, priority, cancellable,
callback, user_data);
}
gboolean
_ostree_fetcher_request_to_membuf_finish (OstreeFetcher *self,
GAsyncResult *result,
GBytes **out_buf,
GError **error)
{
GTask *task;
FetcherRequest *req;
gpointer ret;
g_return_val_if_fail (g_task_is_valid (result, self), FALSE);
g_return_val_if_fail (g_async_result_is_tagged (result, _ostree_fetcher_request_async), FALSE);
task = (GTask*)result;
req = g_task_get_task_data (task);
ret = g_task_propagate_pointer (task, error);
if (!ret)
return FALSE;
g_assert (req->is_membuf);
g_assert (out_buf);
*out_buf = ret;
return TRUE;
}
guint64
_ostree_fetcher_bytes_transferred (OstreeFetcher *self)
{
return self->bytes_transferred;
}
src/libostree/ostree-fetcher.c → src/libostree/ostree-fetcher-soup.c
+37 -142
View File
@@ -32,6 +32,7 @@
#include "libglnx.h"
#include "ostree-fetcher.h"
#include "ostree-fetcher-util.h"
#ifdef HAVE_LIBSOUP_CLIENT_CERTS
#include "ostree-tls-cert-interaction.h"
#endif
@@ -55,6 +56,7 @@ typedef struct {
GError *initialization_error; /* Any failure to load the db */
int tmpdir_dfd;
char *remote_name;
char *tmpdir_name;
GLnxLockFile tmpdir_lock;
int base_tmpdir_dfd;
@@ -62,8 +64,7 @@ typedef struct {
GVariant *extra_headers;
int max_outstanding;
/* Queue for libsoup, see bgo#708591 */
GQueue pending_queue;
/* Our active HTTP requests */
GHashTable *outstanding;
/* Shared across threads; be sure to lock. */
@@ -77,9 +78,6 @@ typedef struct {
} ThreadClosure;
static void
session_thread_process_pending_queue (ThreadClosure *thread_closure);
typedef struct {
volatile int ref_count;
@@ -172,6 +170,8 @@ thread_closure_unref (ThreadClosure *thread_closure)
g_clear_pointer (&thread_closure->oob_error, g_error_free);
g_free (thread_closure->remote_name);
g_slice_free (ThreadClosure, thread_closure);
}
}
@@ -187,18 +187,6 @@ idle_closure_free (IdleClosure *idle_closure)
g_slice_free (IdleClosure, idle_closure);
}
static int
pending_task_compare (gconstpointer a,
gconstpointer b,
gpointer unused)
{
gint priority_a = g_task_get_priority (G_TASK (a));
gint priority_b = g_task_get_priority (G_TASK (b));
return (priority_a == priority_b) ? 0 :
(priority_a < priority_b) ? -1 : 1;
}
static OstreeFetcherPendingURI *
pending_uri_ref (OstreeFetcherPendingURI *pending)
{
@@ -403,30 +391,23 @@ static void
on_request_sent (GObject *object, GAsyncResult *result, gpointer user_data);
static void
session_thread_process_pending_queue (ThreadClosure *thread_closure)
start_pending_request (ThreadClosure *thread_closure,
GTask *task)
{
while (g_queue_peek_head (&thread_closure->pending_queue) != NULL &&
g_hash_table_size (thread_closure->outstanding) < thread_closure->max_outstanding)
{
GTask *task;
OstreeFetcherPendingURI *pending;
GCancellable *cancellable;
OstreeFetcherPendingURI *pending;
GCancellable *cancellable;
task = g_queue_pop_head (&thread_closure->pending_queue);
g_assert_cmpint (g_hash_table_size (thread_closure->outstanding), <, thread_closure->max_outstanding);
pending = g_task_get_task_data (task);
cancellable = g_task_get_cancellable (task);
pending = g_task_get_task_data (task);
cancellable = g_task_get_cancellable (task);
g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending));
soup_request_send_async (pending->request,
cancellable,
on_request_sent,
g_object_ref (task));
g_object_unref (task);
}
g_hash_table_add (thread_closure->outstanding, pending_uri_ref (pending));
soup_request_send_async (pending->request,
cancellable,
on_request_sent,
g_object_ref (task));
}
static void
@@ -547,10 +528,7 @@ session_thread_request_uri (ThreadClosure *thread_closure,
pending->out_tmpfile = tmpfile;
tmpfile = NULL; /* Transfer ownership */
g_queue_insert_sorted (&thread_closure->pending_queue,
g_object_ref (task),
pending_task_compare, NULL);
session_thread_process_pending_queue (thread_closure);
start_pending_request (thread_closure, task);
}
}
@@ -578,11 +556,11 @@ ostree_fetcher_session_thread (gpointer data)
/* XXX: Now that we have mirrorlist support, we could make this even smarter
* by spreading requests across mirrors. */
g_object_get (closure->session, "max-conns-per-host", &max_conns, NULL);
if (max_conns < 8)
if (max_conns < _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS)
{
/* We download a lot of small objects in ostree, so this
* helps a lot. Also matches what most modern browsers do. */
max_conns = 8;
max_conns = _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS;
g_object_set (closure->session,
"max-conns-per-host",
max_conns, NULL);
@@ -600,8 +578,6 @@ ostree_fetcher_session_thread (gpointer data)
* unreference all data related to the SoupSession ourself to ensure
* it's freed in the same thread where it was created. */
g_clear_pointer (&closure->outstanding, g_hash_table_unref);
while (!g_queue_is_empty (&closure->pending_queue))
g_object_unref (g_queue_pop_head (&closure->pending_queue));
g_clear_pointer (&closure->session, g_object_unref);
thread_closure_unref (closure);
@@ -753,12 +729,13 @@ _ostree_fetcher_init (OstreeFetcher *self)
OstreeFetcher *
_ostree_fetcher_new (int tmpdir_dfd,
const char *remote_name,
OstreeFetcherConfigFlags flags)
{
OstreeFetcher *self;
self = g_object_new (OSTREE_TYPE_FETCHER, "config-flags", flags, NULL);
self->thread_closure->remote_name = g_strdup (remote_name);
self->thread_closure->base_tmpdir_dfd = tmpdir_dfd;
return self;
@@ -903,11 +880,6 @@ finish_stream (OstreeFetcherPendingURI *pending,
pending->state = OSTREE_FETCHER_STATE_COMPLETE;
/* Now that we've finished downloading, continue with other queued
* requests.
*/
session_thread_process_pending_queue (pending->thread_closure);
if (!pending->is_membuf)
{
if (stbuf.st_size < pending->content_length)
@@ -935,14 +907,13 @@ on_stream_read (GObject *object,
gpointer user_data);
static void
remove_pending_rerun_queue (OstreeFetcherPendingURI *pending)
remove_pending (OstreeFetcherPendingURI *pending)
{
/* Hold a temporary ref to ensure the reference to
* pending->thread_closure is valid.
*/
pending_uri_ref (pending);
g_hash_table_remove (pending->thread_closure->outstanding, pending);
session_thread_process_pending_queue (pending->thread_closure);
pending_uri_unref (pending);
}
@@ -976,7 +947,7 @@ on_out_splice_complete (GObject *object,
if (local_error)
{
g_task_return_error (task, local_error);
remove_pending_rerun_queue (pending);
remove_pending (pending);
}
g_object_unref (task);
@@ -1018,7 +989,7 @@ on_stream_read (GObject *object,
g_strdup (pending->out_tmpfile),
(GDestroyNotify) g_free);
}
remove_pending_rerun_queue (pending);
remove_pending (pending);
}
else
{
@@ -1057,7 +1028,7 @@ on_stream_read (GObject *object,
if (local_error)
{
g_task_return_error (task, local_error);
remove_pending_rerun_queue (pending);
remove_pending (pending);
}
g_object_unref (task);
@@ -1096,7 +1067,7 @@ on_request_sent (GObject *object,
g_task_return_pointer (task,
g_strdup (pending->out_tmpfile),
(GDestroyNotify) g_free);
remove_pending_rerun_queue (pending);
remove_pending (pending);
goto out;
}
else if (!SOUP_STATUS_IS_SUCCESSFUL (msg->status_code))
@@ -1110,13 +1081,14 @@ on_request_sent (GObject *object,
goto out;
(void) g_input_stream_close (pending->request_body, NULL, NULL);
g_queue_insert_sorted (&pending->thread_closure->pending_queue,
g_object_ref (task), pending_task_compare,
NULL);
remove_pending_rerun_queue (pending);
start_pending_request (pending->thread_closure, task);
}
else
{
g_autofree char *uristring
= soup_uri_to_string (soup_request_get_uri (pending->request), FALSE);
GIOErrorEnum code;
switch (msg->status_code)
{
@@ -1151,6 +1123,10 @@ on_request_sent (GObject *object,
g_prefix_error (&local_error,
"All %u mirrors failed. Last error was: ",
pending->mirrorlist->len);
if (pending->thread_closure->remote_name)
_ostree_fetcher_journal_failure (pending->thread_closure->remote_name,
uristring, local_error->message);
}
goto out;
}
@@ -1204,7 +1180,7 @@ on_request_sent (GObject *object,
if (pending->request_body)
(void) g_input_stream_close (pending->request_body, NULL, NULL);
g_task_return_error (task, local_error);
remove_pending_rerun_queue (pending);
remove_pending (pending);
}
g_object_unref (task);
@@ -1368,84 +1344,3 @@ _ostree_fetcher_bytes_transferred (OstreeFetcher *self)
return ret;
}
void
_ostree_fetcher_uri_free (OstreeFetcherURI *uri)
{
if (uri)
soup_uri_free ((SoupURI*)uri);
}
OstreeFetcherURI *
_ostree_fetcher_uri_parse (const char *str,
GError **error)
{
SoupURI *soupuri = soup_uri_new (str);
if (soupuri == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse uri: %s", str);
return NULL;
}
return (OstreeFetcherURI*)soupuri;
}
static OstreeFetcherURI *
_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri,
gboolean extend,
const char *path)
{
SoupURI *newuri = soup_uri_copy ((SoupURI*)uri);
if (path)
{
if (extend)
{
const char *origpath = soup_uri_get_path ((SoupURI*)uri);
g_autofree char *newpath = g_build_filename (origpath, path, NULL);
soup_uri_set_path (newuri, newpath);
}
else
{
soup_uri_set_path (newuri, path);
}
}
return (OstreeFetcherURI*)newuri;
}
OstreeFetcherURI *
_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri,
const char *path)
{
return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path);
}
OstreeFetcherURI *
_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri,
const char *subpath)
{
return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath);
}
OstreeFetcherURI *
_ostree_fetcher_uri_clone (OstreeFetcherURI *uri)
{
return _ostree_fetcher_uri_new_subpath (uri, NULL);
}
char *
_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri)
{
return g_strdup (soup_uri_get_scheme ((SoupURI*)uri));
}
char *
_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri)
{
return g_strdup (soup_uri_get_path ((SoupURI*)uri));
}
char *
_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri)
{
return soup_uri_to_string ((SoupURI*)uri, FALSE);
}
src/libostree/ostree-fetcher-uri.c
+118
View File
@@ -0,0 +1,118 @@
/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
*
* Copyright (C) 2011,2017 Colin Walters <walters@verbum.org>
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, write to the
* Free Software Foundation, Inc., 59 Temple Place - Suite 330,
* Boston, MA 02111-1307, USA.
*
* Author: Colin Walters <walters@verbum.org>
*/
#include "config.h"
#ifdef HAVE_LIBCURL
#include "ostree-soup-uri.h"
#else
#define LIBSOUP_USE_UNSTABLE_REQUEST_API
#include <libsoup/soup.h>
#include <libsoup/soup-requester.h>
#include <libsoup/soup-request-http.h>
#endif
#include "ostree-fetcher.h"
#include "libglnx.h"
void
_ostree_fetcher_uri_free (OstreeFetcherURI *uri)
{
if (uri)
soup_uri_free ((SoupURI*)uri);
}
OstreeFetcherURI *
_ostree_fetcher_uri_parse (const char *str,
GError **error)
{
SoupURI *soupuri = soup_uri_new (str);
if (soupuri == NULL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Failed to parse uri: %s", str);
return NULL;
}
return (OstreeFetcherURI*)soupuri;
}
static OstreeFetcherURI *
_ostree_fetcher_uri_new_path_internal (OstreeFetcherURI *uri,
gboolean extend,
const char *path)
{
SoupURI *newuri = soup_uri_copy ((SoupURI*)uri);
if (path)
{
if (extend)
{
const char *origpath = soup_uri_get_path ((SoupURI*)uri);
g_autofree char *newpath = g_build_filename (origpath, path, NULL);
soup_uri_set_path (newuri, newpath);
}
else
{
soup_uri_set_path (newuri, path);
}
}
return (OstreeFetcherURI*)newuri;
}
OstreeFetcherURI *
_ostree_fetcher_uri_new_path (OstreeFetcherURI *uri,
const char *path)
{
return _ostree_fetcher_uri_new_path_internal (uri, FALSE, path);
}
OstreeFetcherURI *
_ostree_fetcher_uri_new_subpath (OstreeFetcherURI *uri,
const char *subpath)
{
return _ostree_fetcher_uri_new_path_internal (uri, TRUE, subpath);
}
OstreeFetcherURI *
_ostree_fetcher_uri_clone (OstreeFetcherURI *uri)
{
return _ostree_fetcher_uri_new_subpath (uri, NULL);
}
char *
_ostree_fetcher_uri_get_scheme (OstreeFetcherURI *uri)
{
return g_strdup (soup_uri_get_scheme ((SoupURI*)uri));
}
char *
_ostree_fetcher_uri_get_path (OstreeFetcherURI *uri)
{
return g_strdup (soup_uri_get_path ((SoupURI*)uri));
}
char *
_ostree_fetcher_uri_to_string (OstreeFetcherURI *uri)
{
return soup_uri_to_string ((SoupURI*)uri, FALSE);
}
src/libostree/ostree-fetcher-util.c
+24
View File
@@ -23,6 +23,10 @@
#include <gio/gfiledescriptorbased.h>
#include <gio/gunixoutputstream.h>
#ifdef HAVE_LIBSYSTEMD
#include <systemd/sd-journal.h>
#endif
#include "ostree-fetcher-util.h"
#include "otutil.h"
@@ -122,3 +126,23 @@ _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
out_contents, max_size,
cancellable, error);
}
#define OSTREE_HTTP_FAILURE_ID SD_ID128_MAKE(f0,2b,ce,89,a5,4e,4e,fa,b3,a9,4a,79,7d,26,20,4a)
void
_ostree_fetcher_journal_failure (const char *remote_name,
const char *url,
const char *msg)
{
#ifdef HAVE_LIBSYSTEMD
/* Sanity - we don't want to log this when doing local/file pulls */
if (!remote_name)
return;
sd_journal_send ("MESSAGE=libostree HTTP error from remote %s for <%s>: %s",
remote_name, url, msg,
"MESSAGE_ID=" SD_ID128_FORMAT_STR, SD_ID128_FORMAT_VAL(OSTREE_HTTP_FAILURE_ID),
"OSTREE_REMOTE=%s", remote_name,
"OSTREE_URL=%s", url,
NULL);
#endif
}
src/libostree/ostree-fetcher-util.h
+6
View File
@@ -44,6 +44,12 @@ gboolean _ostree_fetcher_request_uri_to_membuf (OstreeFetcher *fetcher,
guint64 max_size,
GCancellable *cancellable,
GError **error);
void _ostree_fetcher_journal_failure (const char *remote_name,
const char *url,
const char *msg);
G_END_DECLS
#endif
src/libostree/ostree-fetcher.h
+1
View File
@@ -82,6 +82,7 @@ _ostree_fetcher_uri_to_string (OstreeFetcherURI *uri);
GType _ostree_fetcher_get_type (void) G_GNUC_CONST;
OstreeFetcher *_ostree_fetcher_new (int tmpdir_dfd,
const char *remote_name,
OstreeFetcherConfigFlags flags);
int _ostree_fetcher_get_dfd (OstreeFetcher *fetcher);
src/libostree/ostree-libarchive-private.h
+3
View File
@@ -37,6 +37,9 @@ GLNX_DEFINE_CLEANUP_FUNCTION (void *, flatpak_local_free_write_archive, archive_
GLNX_DEFINE_CLEANUP_FUNCTION (void *, flatpak_local_free_read_archive, archive_read_free)
#define ot_cleanup_read_archive __attribute__((cleanup (flatpak_local_free_read_archive)))
#else
#define ot_cleanup_write_archive
#define ot_cleanup_read_archive
#endif
G_END_DECLS
src/libostree/ostree-repo-checkout.c
+88 -50
View File
@@ -201,8 +201,16 @@ checkout_file_from_input_at (OstreeRepo *self,
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (res == -1)
{
glnx_set_error_from_errno (error);
goto out;
if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)
{
ret = TRUE;
goto out;
}
else
{
glnx_set_error_from_errno (error);
goto out;
}
}
if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
@@ -240,6 +248,11 @@ checkout_file_from_input_at (OstreeRepo *self,
while (G_UNLIKELY (fd == -1 && errno == EINTR));
if (fd == -1)
{
if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)
{
ret = TRUE;
goto out;
}
glnx_set_error_from_errno (error);
goto out;
}
@@ -332,6 +345,12 @@ checkout_file_unioning_from_input_at (OstreeRepo *repo,
return ret;
}
typedef enum {
HARDLINK_RESULT_NOT_SUPPORTED,
HARDLINK_RESULT_SKIP_EXISTED,
HARDLINK_RESULT_LINKED
} HardlinkResult;
static gboolean
checkout_file_hardlink (OstreeRepo *self,
OstreeRepoCheckoutAtOptions *options,
@@ -339,28 +358,32 @@ checkout_file_hardlink (OstreeRepo *self,
int destination_dfd,
const char *destination_name,
gboolean allow_noent,
gboolean *out_was_supported,
HardlinkResult *out_result,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gboolean ret_was_supported = FALSE;
HardlinkResult ret_result = HARDLINK_RESULT_NOT_SUPPORTED;
int srcfd = (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER) ?
self->objects_dir_fd : self->uncompressed_objects_dir_fd;
again:
if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) != -1)
ret_was_supported = TRUE;
if (linkat (srcfd, loose_path, destination_dfd, destination_name, 0) == 0)
ret_result = HARDLINK_RESULT_LINKED;
else if (!options->no_copy_fallback && (errno == EMLINK || errno == EXDEV || errno == EPERM))
{
/* EMLINK, EXDEV and EPERM shouldn't be fatal; we just can't do the
* optimization of hardlinking instead of copying.
*/
ret_was_supported = FALSE;
}
else if (allow_noent && errno == ENOENT)
{
ret_was_supported = FALSE;
}
else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES)
{
/* In this mode, we keep existing content. Distinguish this case though to
* avoid inserting into the devino cache.
*/
ret_result = HARDLINK_RESULT_SKIP_EXISTED;
}
else if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES)
{
@@ -370,7 +393,7 @@ checkout_file_hardlink (OstreeRepo *self,
* the same file, then rename() does nothing, and returns a
* success status."
*
* So we can't make this atomic.
* So we can't make this atomic.
*/
(void) unlinkat (destination_dfd, destination_name, 0);
goto again;
@@ -379,14 +402,12 @@ checkout_file_hardlink (OstreeRepo *self,
{
g_prefix_error (error, "Hardlinking %s to %s: ", loose_path, destination_name);
glnx_set_error_from_errno (error);
goto out;
return FALSE;
}
ret = TRUE;
if (out_was_supported)
*out_was_supported = ret_was_supported;
out:
return ret;
if (out_result)
*out_result = ret_result;
return TRUE;
}
static gboolean
@@ -439,7 +460,7 @@ checkout_one_file_at (OstreeRepo *repo,
}
else
{
gboolean did_hardlink = FALSE;
HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED;
/* Try to do a hardlink first, if it's a regular file. This also
* traverses all parent repos.
*/
@@ -469,11 +490,11 @@ checkout_one_file_at (OstreeRepo *repo,
options,
loose_path_buf,
destination_dfd, destination_name,
TRUE, &did_hardlink,
TRUE, &hardlink_res,
cancellable, error))
goto out;
if (did_hardlink && options->devino_to_csum_cache)
if (hardlink_res == HARDLINK_RESULT_LINKED && options->devino_to_csum_cache)
{
struct stat stbuf;
OstreeDevIno *key;
@@ -492,13 +513,13 @@ checkout_one_file_at (OstreeRepo *repo,
g_hash_table_add ((GHashTable*)options->devino_to_csum_cache, key);
}
if (did_hardlink)
if (hardlink_res != HARDLINK_RESULT_NOT_SUPPORTED)
break;
}
current_repo = current_repo->parent_repo;
}
need_copy = !did_hardlink;
need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED);
}
can_cache = (options->enable_uncompressed_cache
@@ -514,7 +535,7 @@ checkout_one_file_at (OstreeRepo *repo,
&& repo->mode == OSTREE_REPO_MODE_ARCHIVE_Z2
&& options->mode == OSTREE_REPO_CHECKOUT_MODE_USER)
{
gboolean did_hardlink;
HardlinkResult hardlink_res = HARDLINK_RESULT_NOT_SUPPORTED;
if (!ostree_repo_load_file (repo, checksum, &input, NULL, NULL,
cancellable, error))
@@ -560,19 +581,20 @@ checkout_one_file_at (OstreeRepo *repo,
if (!checkout_file_hardlink (repo, options, loose_path_buf,
destination_dfd, destination_name,
FALSE, &did_hardlink,
FALSE, &hardlink_res,
cancellable, error))
{
g_prefix_error (error, "Using new cached uncompressed hardlink of %s to %s: ", checksum, destination_name);
goto out;
}
need_copy = !did_hardlink;
need_copy = (hardlink_res == HARDLINK_RESULT_NOT_SUPPORTED);
}
/* Fall back to copy if we couldn't hardlink */
if (need_copy)
{
g_assert (!options->no_copy_fallback);
if (!ostree_repo_load_file (repo, checksum, &input, NULL, &xattrs,
cancellable, error))
goto out;
@@ -641,6 +663,8 @@ checkout_tree_at (OstreeRepo *self,
gboolean did_exist = FALSE;
glnx_fd_close int destination_dfd = -1;
int res;
struct stat repo_dfd_stat;
struct stat destination_stat;
g_autoptr(GVariant) xattrs = NULL;
g_autoptr(GFileEnumerator) dir_enum = NULL;
@@ -653,7 +677,9 @@ checkout_tree_at (OstreeRepo *self,
while (G_UNLIKELY (res == -1 && errno == EINTR));
if (res == -1)
{
if (errno == EEXIST && options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES)
if (errno == EEXIST &&
(options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES
|| options->overwrite_mode == OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES))
did_exist = TRUE;
else
{
@@ -666,6 +692,25 @@ checkout_tree_at (OstreeRepo *self,
&destination_dfd, error))
goto out;
if (fstat (self->repo_dir_fd, &repo_dfd_stat) < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
if (fstat (destination_dfd, &destination_stat) < 0)
{
glnx_set_error_from_errno (error);
goto out;
}
if (options->no_copy_fallback && repo_dfd_stat.st_dev != destination_stat.st_dev)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Unable to do hardlink checkout across devices (src=%"G_GUINT64_FORMAT" destination=%"G_GUINT64_FORMAT")",
(guint64)repo_dfd_stat.st_dev, (guint64)destination_stat.st_dev);
goto out;
}
/* Set the xattrs now, so any derived labeling works */
if (!did_exist && options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
{
@@ -902,10 +947,6 @@ ostree_repo_checkout_at (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GFile) commit_root = NULL;
g_autoptr(GFile) target_dir = NULL;
g_autoptr(GFileInfo) target_info = NULL;
OstreeRepoCheckoutAtOptions default_options = { 0, };
if (!options)
@@ -914,33 +955,33 @@ ostree_repo_checkout_at (OstreeRepo *self,
options = &default_options;
}
commit_root = (GFile*) _ostree_repo_file_new_for_commit (self, commit, error);
g_autoptr(GFile) commit_root = (GFile*) _ostree_repo_file_new_for_commit (self, commit, error);
if (!commit_root)
goto out;
return FALSE;
if (!ostree_repo_file_ensure_resolved ((OstreeRepoFile*)commit_root, error))
goto out;
return FALSE;
g_autoptr(GFile) target_dir = NULL;
if (options->subpath && strcmp (options->subpath, "/") != 0)
target_dir = g_file_get_child (commit_root, options->subpath);
else
target_dir = g_object_ref (commit_root);
target_info = g_file_query_info (target_dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
g_autoptr(GFileInfo) target_info =
g_file_query_info (target_dir, OSTREE_GIO_FAST_QUERYINFO,
G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
cancellable, error);
if (!target_info)
goto out;
return FALSE;
if (!checkout_tree_at (self, options,
destination_dfd,
destination_path,
(OstreeRepoFile*)target_dir, target_info,
cancellable, error))
goto out;
return FALSE;
ret = TRUE;
out:
return ret;
return TRUE;
}
static guint
@@ -994,16 +1035,15 @@ ostree_repo_checkout_gc (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GHashTable) to_clean_dirs = NULL;
GHashTableIter iter;
gpointer key, value;
g_mutex_lock (&self->cache_lock);
to_clean_dirs = self->updated_uncompressed_dirs;
self->updated_uncompressed_dirs = g_hash_table_new (NULL, NULL);
g_mutex_unlock (&self->cache_lock);
GHashTableIter iter;
gpointer key, value;
if (to_clean_dirs)
g_hash_table_iter_init (&iter, to_clean_dirs);
while (to_clean_dirs && g_hash_table_iter_next (&iter, &key, &value))
@@ -1013,7 +1053,7 @@ ostree_repo_checkout_gc (OstreeRepo *self,
if (!glnx_dirfd_iterator_init_at (self->uncompressed_objects_dir_fd, objdir_name, FALSE,
&dfd_iter, error))
goto out;
return FALSE;
while (TRUE)
{
@@ -1021,14 +1061,14 @@ ostree_repo_checkout_gc (OstreeRepo *self,
struct stat stbuf;
if (!glnx_dirfd_iterator_next_dent (&dfd_iter, &dent, cancellable, error))
goto out;
return FALSE;
if (dent == NULL)
break;
if (fstatat (dfd_iter.fd, dent->d_name, &stbuf, AT_SYMLINK_NOFOLLOW) != 0)
{
glnx_set_error_from_errno (error);
goto out;
return FALSE;
}
if (stbuf.st_nlink == 1)
@@ -1036,13 +1076,11 @@ ostree_repo_checkout_gc (OstreeRepo *self,
if (unlinkat (dfd_iter.fd, dent->d_name, 0) != 0)
{
glnx_set_error_from_errno (error);
goto out;
return FALSE;
}
}
}
}
ret = TRUE;
out:
return ret;
return TRUE;
}
src/libostree/ostree-repo-commit.c
+27 -1
View File
@@ -110,6 +110,30 @@ write_file_metadata_to_xattr (int fd,
return TRUE;
}
/* See https://github.com/ostreedev/ostree/pull/698 */
#ifdef WITH_SMACK
#define XATTR_NAME_SMACK "security.SMACK64"
#endif
static void
ot_security_smack_reset_dfd_name (int dfd, const char *name)
{
#ifdef WITH_SMACK
char buf[PATH_MAX];
/* See glnx-xattrs.c */
snprintf (buf, sizeof (buf), "/proc/self/fd/%d/%s", dfd, name);
(void) lremovexattr (buf, XATTR_NAME_SMACK);
#endif
}
static void
ot_security_smack_reset_fd (int fd)
{
#ifdef WITH_SMACK
(void) fremovexattr (fd, XATTR_NAME_SMACK);
#endif
}
gboolean
_ostree_repo_commit_loose_final (OstreeRepo *self,
const char *checksum,
@@ -221,6 +245,7 @@ commit_loose_object_trusted (OstreeRepo *self,
if (xattrs != NULL)
{
ot_security_smack_reset_dfd_name (self->tmp_dir_fd, temp_filename);
if (!glnx_dfd_name_set_all_xattrs (self->tmp_dir_fd, temp_filename,
xattrs, cancellable, error))
goto out;
@@ -252,6 +277,7 @@ commit_loose_object_trusted (OstreeRepo *self,
if (xattrs)
{
ot_security_smack_reset_fd (fd);
if (!glnx_fd_set_all_xattrs (fd, xattrs, cancellable, error))
goto out;
}
@@ -719,7 +745,7 @@ write_object (OstreeRepo *self,
if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_REGULAR)
{
zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, 9);
zlib_compressor = (GConverter*)g_zlib_compressor_new (G_ZLIB_COMPRESSOR_FORMAT_RAW, self->zlib_compression_level);
compressed_out_stream = g_converter_output_stream_new (temp_out, zlib_compressor);
/* Don't close the base; we'll do that later */
g_filter_output_stream_set_close_base_stream ((GFilterOutputStream*)compressed_out_stream, FALSE);
src/libostree/ostree-repo-private.h
+12
View File
@@ -32,6 +32,17 @@ G_BEGIN_DECLS
#define _OSTREE_SUMMARY_CACHE_DIR "summaries"
#define _OSTREE_CACHE_DIR "cache"
#define _OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS 8
#define _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS 2
/* In most cases, writing to disk should be much faster than
* fetching from the network, so we shouldn't actually hit
* this. But if using pipelining and e.g. pulling over LAN
* (or writing to slow media), we can have a runaway
* situation towards EMFILE.
* */
#define _OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS 16
typedef enum {
OSTREE_REPO_TEST_ERROR_PRE_COMMIT = (1 << 0)
} OstreeRepoTestErrorFlags;
@@ -89,6 +100,7 @@ struct OstreeRepo {
GError *writable_error;
gboolean in_transaction;
gboolean disable_fsync;
guint zlib_compression_level;
GHashTable *loose_object_devino_hash;
GHashTable *updated_uncompressed_dirs;
GHashTable *object_sizes;
src/libostree/ostree-repo-pull.c
+390 -79
View File
@@ -26,7 +26,7 @@
#include "ostree.h"
#include "otutil.h"
#ifdef HAVE_LIBSOUP
#ifdef HAVE_LIBCURL_OR_LIBSOUP
#include "ostree-core-private.h"
#include "ostree-repo-private.h"
@@ -81,6 +81,10 @@ typedef struct {
GHashTable *scanned_metadata; /* Maps object name to itself */
GHashTable *requested_metadata; /* Maps object name to itself */
GHashTable *requested_content; /* Maps checksum to itself */
GHashTable *requested_fallback_content; /* Maps checksum to itself */
GHashTable *pending_fetch_metadata; /* Map<ObjectName,FetchObjectData> */
GHashTable *pending_fetch_content; /* Map<checksum,FetchObjectData> */
GHashTable *pending_fetch_deltaparts; /* Set<FetchStaticDeltaData> */
guint n_outstanding_metadata_fetches;
guint n_outstanding_metadata_write_requests;
guint n_outstanding_content_fetches;
@@ -88,11 +92,14 @@ typedef struct {
guint n_outstanding_deltapart_fetches;
guint n_outstanding_deltapart_write_requests;
guint n_total_deltaparts;
guint n_total_delta_fallbacks;
guint64 fetched_deltapart_size; /* How much of the delta we have now */
guint64 total_deltapart_size;
guint64 total_deltapart_usize;
gint n_requested_metadata;
gint n_requested_content;
guint n_fetched_deltaparts;
guint n_fetched_deltapart_fallbacks;
guint n_fetched_metadata;
guint n_fetched_content;
@@ -133,6 +140,10 @@ typedef struct {
OtPullData *pull_data;
GVariant *objects;
char *expected_checksum;
char *from_revision;
char *to_revision;
guint i;
guint64 size;
} FetchStaticDeltaData;
typedef struct {
@@ -142,6 +153,10 @@ typedef struct {
guint recursion_depth;
} ScanObjectQueueData;
static void start_fetch (OtPullData *pull_data, FetchObjectData *fetch);
static void start_fetch_deltapart (OtPullData *pull_data,
FetchStaticDeltaData *fetch);
static gboolean fetcher_queue_is_full (OtPullData *pull_data);
static void queue_scan_one_metadata_object (OtPullData *pull_data,
const char *csum,
OstreeObjectType objtype,
@@ -199,6 +214,7 @@ update_progress (gpointer user_data)
ostree_async_progress_set_uint (pull_data->progress, "outstanding-writes", outstanding_writes);
ostree_async_progress_set_uint (pull_data->progress, "fetched", fetched);
ostree_async_progress_set_uint (pull_data->progress, "requested", requested);
ostree_async_progress_set_uint (pull_data->progress, "scanning", g_queue_is_empty (&pull_data->scan_object_queue) ? 0 : 1);
ostree_async_progress_set_uint (pull_data->progress, "scanned-metadata", n_scanned_metadata);
ostree_async_progress_set_uint64 (pull_data->progress, "bytes-transferred", bytes_transferred);
ostree_async_progress_set_uint64 (pull_data->progress, "start-time", start_time);
@@ -208,6 +224,12 @@ update_progress (gpointer user_data)
pull_data->n_fetched_deltaparts);
ostree_async_progress_set_uint (pull_data->progress, "total-delta-parts",
pull_data->n_total_deltaparts);
ostree_async_progress_set_uint (pull_data->progress, "fetched-delta-fallbacks",
pull_data->n_fetched_deltapart_fallbacks);
ostree_async_progress_set_uint (pull_data->progress, "total-delta-fallbacks",
pull_data->n_total_delta_fallbacks);
ostree_async_progress_set_uint64 (pull_data->progress, "fetched-delta-part-size",
pull_data->fetched_deltapart_size);
ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-size",
pull_data->total_deltapart_size);
ostree_async_progress_set_uint64 (pull_data->progress, "total-delta-part-usize",
@@ -271,6 +293,88 @@ check_outstanding_requests_handle_error (OtPullData *pull_data,
g_error_free (error);
}
}
else
{
GHashTableIter hiter;
gpointer key, value;
/* We may have just completed an async fetch operation. Now we look at
* possibly enqueuing more requests. The goal of queuing is to both avoid
* overloading the fetcher backend with HTTP requests, but also to
* prioritize metadata fetches over content, so we have accurate
* reporting. Hence here, we process metadata fetches first.
*/
/* Try filling the queue with metadata we need to fetch */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_metadata);
while (!fetcher_queue_is_full (pull_data) &&
g_hash_table_iter_next (&hiter, &key, &value))
{
GVariant *objname = key;
FetchObjectData *fetch = value;
/* Steal both key and value */
g_hash_table_iter_steal (&hiter);
/* This takes ownership of the value */
start_fetch (pull_data, fetch);
/* And unref the key */
g_variant_unref (objname);
}
/* Now, process deltapart requests */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_deltaparts);
while (!fetcher_queue_is_full (pull_data) &&
g_hash_table_iter_next (&hiter, &key, &value))
{
FetchStaticDeltaData *fetch = key;
g_hash_table_iter_steal (&hiter);
/* Takes ownership */
start_fetch_deltapart (pull_data, fetch);
}
/* Next, fill the queue with content */
g_hash_table_iter_init (&hiter, pull_data->pending_fetch_content);
while (!fetcher_queue_is_full (pull_data) &&
g_hash_table_iter_next (&hiter, &key, &value))
{
char *checksum = key;
FetchObjectData *fetch = value;
/* Steal both key and value */
g_hash_table_iter_steal (&hiter);
/* This takes ownership of the value */
start_fetch (pull_data, fetch);
/* And unref the key */
g_free (checksum);
}
}
}
/* We have a total-request limit, as well has a hardcoded max of 2 for delta
* parts. The logic for the delta one is that processing them is expensive, and
* doing multiple simultaneously could risk space/memory on smaller devices. We
* also throttle on outstanding writes in case fetches are faster.
*/
static gboolean
fetcher_queue_is_full (OtPullData *pull_data)
{
const gboolean fetch_full =
((pull_data->n_outstanding_metadata_fetches +
pull_data->n_outstanding_content_fetches +
pull_data->n_outstanding_deltapart_fetches) ==
_OSTREE_MAX_OUTSTANDING_FETCHER_REQUESTS);
const gboolean deltas_full =
(pull_data->n_outstanding_deltapart_fetches ==
_OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
const gboolean writes_full =
((pull_data->n_outstanding_metadata_write_requests +
pull_data->n_outstanding_content_write_requests +
pull_data->n_outstanding_deltapart_write_requests) >=
_OSTREE_MAX_OUTSTANDING_WRITE_REQUESTS);
return fetch_full || deltas_full || writes_full;
}
static gboolean
@@ -685,6 +789,9 @@ content_fetch_on_write_complete (GObject *object,
}
pull_data->n_fetched_content++;
/* Was this a delta fallback? */
if (g_hash_table_remove (pull_data->requested_fallback_content, expected_checksum))
pull_data->n_fetched_deltapart_fallbacks++;
out:
pull_data->n_outstanding_content_write_requests--;
check_outstanding_requests_handle_error (pull_data, local_error);
@@ -942,6 +1049,8 @@ fetch_static_delta_data_free (gpointer data)
FetchStaticDeltaData *fetch_data = data;
g_free (fetch_data->expected_checksum);
g_variant_unref (fetch_data->objects);
g_free (fetch_data->from_revision);
g_free (fetch_data->to_revision);
g_free (fetch_data);
}
@@ -1035,14 +1144,22 @@ process_verify_result (OtPullData *pull_data,
GError **error)
{
if (result == NULL)
return FALSE;
{
g_prefix_error (error, "Commit %s: ", checksum);
return FALSE;
}
/* Allow callers to output the results immediately. */
g_signal_emit_by_name (pull_data->repo,
"gpg-verify-result",
checksum, result);
return ostree_gpg_verify_result_require_valid_signature (result, error);
if (!ostree_gpg_verify_result_require_valid_signature (result, error))
{
g_prefix_error (error, "Commit %s: ", checksum);
return FALSE;
}
return TRUE;
}
static gboolean
@@ -1060,8 +1177,9 @@ gpg_verify_unwritten_commit (OtPullData *pull_data,
if (!detached_metadata)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"No detached metadata found for GPG verification");
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Commit %s: no detached metadata found for GPG verification",
checksum);
return FALSE;
}
@@ -1334,41 +1452,11 @@ enqueue_one_object_request (OtPullData *pull_data,
gboolean is_detached_meta,
gboolean object_is_stored)
{
g_autofree char *obj_subpath = NULL;
gboolean is_meta;
FetchObjectData *fetch_data;
guint64 *expected_max_size_p;
guint64 expected_max_size;
GPtrArray *mirrorlist = NULL;
g_debug ("queuing fetch of %s.%s%s", checksum,
ostree_object_type_to_string (objtype),
is_detached_meta ? " (detached)" : "");
if (is_detached_meta)
{
char buf[_OSTREE_LOOSE_PATH_MAX];
_ostree_loose_path (buf, checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode);
obj_subpath = g_build_filename ("objects", buf, NULL);
mirrorlist = pull_data->meta_mirrorlist;
}
else
{
obj_subpath = _ostree_get_relative_object_path (checksum, objtype, TRUE);
mirrorlist = pull_data->content_mirrorlist;
}
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
if (is_meta)
{
pull_data->n_outstanding_metadata_fetches++;
pull_data->n_requested_metadata++;
}
else
{
pull_data->n_outstanding_content_fetches++;
pull_data->n_requested_content++;
}
fetch_data = g_new0 (FetchObjectData, 1);
fetch_data->pull_data = pull_data;
fetch_data->object = ostree_object_name_serialize (checksum, objtype);
@@ -1376,10 +1464,80 @@ enqueue_one_object_request (OtPullData *pull_data,
fetch_data->is_detached_meta = is_detached_meta;
fetch_data->object_is_stored = object_is_stored;
expected_max_size_p = is_detached_meta ? NULL : g_hash_table_lookup (pull_data->expected_commit_sizes, checksum);
if (is_meta)
pull_data->n_requested_metadata++;
else
pull_data->n_requested_content++;
/* Are too many requests are in flight? */
if (fetcher_queue_is_full (pull_data))
{
g_debug ("queuing fetch of %s.%s%s", checksum,
ostree_object_type_to_string (objtype),
is_detached_meta ? " (detached)" : "");
if (is_meta)
{
GVariant *objname = ostree_object_name_serialize (checksum, objtype);
g_hash_table_insert (pull_data->pending_fetch_metadata, objname, fetch_data);
}
else
{
g_hash_table_insert (pull_data->pending_fetch_content, g_strdup (checksum), fetch_data);
}
}
else
{
start_fetch (pull_data, fetch_data);
}
}
static void
start_fetch (OtPullData *pull_data,
FetchObjectData *fetch)
{
gboolean is_meta;
g_autofree char *obj_subpath = NULL;
guint64 *expected_max_size_p;
guint64 expected_max_size;
const char *expected_checksum;
OstreeObjectType objtype;
GPtrArray *mirrorlist = NULL;
ostree_object_name_deserialize (fetch->object, &expected_checksum, &objtype);
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
g_debug ("starting fetch of %s.%s%s", expected_checksum,
ostree_object_type_to_string (objtype),
fetch->is_detached_meta ? " (detached)" : "");
is_meta = OSTREE_OBJECT_TYPE_IS_META (objtype);
if (is_meta)
pull_data->n_outstanding_metadata_fetches++;
else
pull_data->n_outstanding_content_fetches++;
/* Override the path if we're trying to fetch the .commitmeta file first */
if (fetch->is_detached_meta)
{
char buf[_OSTREE_LOOSE_PATH_MAX];
_ostree_loose_path (buf, expected_checksum, OSTREE_OBJECT_TYPE_COMMIT_META, pull_data->remote_mode);
obj_subpath = g_build_filename ("objects", buf, NULL);
mirrorlist = pull_data->meta_mirrorlist;
}
else
{
obj_subpath = _ostree_get_relative_object_path (expected_checksum, objtype, TRUE);
mirrorlist = pull_data->content_mirrorlist;
}
/* We may have determined maximum sizes from the summary file content; if so,
* honor it. Otherwise, metadata has a baseline max size.
*/
expected_max_size_p = fetch->is_detached_meta ? NULL : g_hash_table_lookup (pull_data->expected_commit_sizes, expected_checksum);
if (expected_max_size_p)
expected_max_size = *expected_max_size_p;
else if (is_meta)
else if (OSTREE_OBJECT_TYPE_IS_META (objtype))
expected_max_size = OSTREE_MAX_METADATA_SIZE;
else
expected_max_size = 0;
@@ -1389,7 +1547,7 @@ enqueue_one_object_request (OtPullData *pull_data,
is_meta ? OSTREE_REPO_PULL_METADATA_PRIORITY
: OSTREE_REPO_PULL_CONTENT_PRIORITY,
pull_data->cancellable,
is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data);
is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch);
}
static gboolean
@@ -1445,15 +1603,10 @@ process_one_static_delta_fallback (OtPullData *pull_data,
compressed_size = maybe_swap_endian_u64 (delta_byteswap, compressed_size);
uncompressed_size = maybe_swap_endian_u64 (delta_byteswap, uncompressed_size);
pull_data->n_total_delta_fallbacks += 1;
pull_data->total_deltapart_size += compressed_size;
pull_data->total_deltapart_usize += uncompressed_size;
if (pull_data->dry_run)
{
ret = TRUE;
goto out;
}
objtype = (OstreeObjectType)objtype_y;
checksum = ostree_checksum_from_bytes_v (csum_v);
@@ -1462,28 +1615,37 @@ process_one_static_delta_fallback (OtPullData *pull_data,
cancellable, error))
goto out;
if (is_stored)
pull_data->fetched_deltapart_size += compressed_size;
if (pull_data->dry_run)
{
ret = TRUE;
goto out;
}
if (!is_stored)
{
{
/* The delta compiler never did this, there's no reason to support it */
if (OSTREE_OBJECT_TYPE_IS_META (objtype))
{
g_autoptr(GVariant) objname = ostree_object_name_serialize (checksum, objtype);
if (!g_hash_table_lookup (pull_data->requested_metadata, objname))
{
gboolean do_fetch_detached;
g_hash_table_add (pull_data->requested_metadata, g_variant_ref (objname));
do_fetch_detached = (objtype == OSTREE_OBJECT_TYPE_COMMIT);
enqueue_one_object_request (pull_data, checksum, objtype, NULL, do_fetch_detached, FALSE);
checksum = NULL; /* Transfer ownership */
}
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Found metadata object as fallback: %s.%s", checksum,
ostree_object_type_to_string (objtype));
goto out;
}
else
{
if (!g_hash_table_lookup (pull_data->requested_content, checksum))
{
/* Mark this as requested, like we do in the non-delta path */
g_hash_table_add (pull_data->requested_content, checksum);
/* But also record it's a delta fallback object, so we can account
* for it as logically part of the delta fetch.
*/
g_hash_table_add (pull_data->requested_fallback_content, g_strdup (checksum));
enqueue_one_object_request (pull_data, checksum, OSTREE_OBJECT_TYPE_FILE, NULL, FALSE, FALSE);
checksum = NULL; /* Transfer ownership */
checksum = NULL; /* We transferred ownership to the requested_content hash */
}
}
}
@@ -1493,6 +1655,22 @@ process_one_static_delta_fallback (OtPullData *pull_data,
return ret;
}
static void
start_fetch_deltapart (OtPullData *pull_data,
FetchStaticDeltaData *fetch)
{
g_autofree char *deltapart_path = _ostree_get_relative_static_delta_part_path (fetch->from_revision, fetch->to_revision, fetch->i);
pull_data->n_outstanding_deltapart_fetches++;
g_assert_cmpint (pull_data->n_outstanding_deltapart_fetches, <=, _OSTREE_MAX_OUTSTANDING_DELTAPART_REQUESTS);
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
pull_data->content_mirrorlist,
deltapart_path, fetch->size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
fetch);
}
static gboolean
process_one_static_delta (OtPullData *pull_data,
const char *from_revision,
@@ -1618,11 +1796,15 @@ process_one_static_delta (OtPullData *pull_data,
cancellable, error))
goto out;
pull_data->total_deltapart_size += size;
pull_data->total_deltapart_usize += usize;
if (have_all)
{
g_debug ("Have all objects from static delta %s-%s part %u",
from_revision ? from_revision : "empty", to_revision,
i);
pull_data->fetched_deltapart_size += size;
pull_data->n_fetched_deltaparts++;
continue;
}
@@ -1636,16 +1818,17 @@ process_one_static_delta (OtPullData *pull_data,
inline_part_bytes = g_variant_get_data_as_bytes (part_datav);
}
pull_data->total_deltapart_size += size;
pull_data->total_deltapart_usize += usize;
if (pull_data->dry_run)
continue;
fetch_data = g_new0 (FetchStaticDeltaData, 1);
fetch_data->from_revision = g_strdup (from_revision);
fetch_data->to_revision = g_strdup (to_revision);
fetch_data->pull_data = pull_data;
fetch_data->objects = g_variant_ref (objects);
fetch_data->expected_checksum = ostree_checksum_from_bytes_v (csum_v);
fetch_data->size = size;
fetch_data->i = i;
if (inline_part_bytes != NULL)
{
@@ -1669,14 +1852,12 @@ process_one_static_delta (OtPullData *pull_data,
}
else
{
_ostree_fetcher_request_to_tmpfile (pull_data->fetcher,
pull_data->content_mirrorlist,
deltapart_path, size,
OSTREE_FETCHER_DEFAULT_PRIORITY,
pull_data->cancellable,
static_deltapart_fetch_on_complete,
fetch_data);
pull_data->n_outstanding_deltapart_fetches++;
if (!fetcher_queue_is_full (pull_data))
start_fetch_deltapart (pull_data, fetch_data);
else
{
g_hash_table_add (pull_data->pending_fetch_deltaparts, fetch_data);
}
}
}
@@ -1685,6 +1866,87 @@ process_one_static_delta (OtPullData *pull_data,
return ret;
}
/* Loop over the static delta data we got from the summary,
* and find the newest commit for @out_from_revision that
* goes to @to_revision.
*/
static gboolean
get_best_static_delta_start_for (OtPullData *pull_data,
const char *to_revision,
char **out_from_revision,
GCancellable *cancellable,
GError **error)
{
GHashTableIter hiter;
gpointer hkey, hvalue;
/* Array<char*> of possible from checksums */
g_autoptr(GPtrArray) candidates = g_ptr_array_new_with_free_func (g_free);
const char *newest_candidate = NULL;
guint64 newest_candidate_timestamp;
g_assert (pull_data->summary_deltas_checksums != NULL);
g_hash_table_iter_init (&hiter, pull_data->summary_deltas_checksums);
/* Loop over all deltas known from the summary file,
* finding ones which go to to_revision */
while (g_hash_table_iter_next (&hiter, &hkey, &hvalue))
{
const char *delta_name = hkey;
g_autofree char *cur_from_rev = NULL;
g_autofree char *cur_to_rev = NULL;
/* Gracefully handle corrupted (or malicious) summary files */
if (!_ostree_parse_delta_name (delta_name, &cur_from_rev, &cur_to_rev, error))
return FALSE;
/* Is this the checksum we want? */
if (strcmp (cur_to_rev, to_revision) != 0)
continue;
if (cur_from_rev)
g_ptr_array_add (candidates, g_steal_pointer (&cur_from_rev));
}
/* Loop over our candidates, find the newest one */
for (guint i = 0; i < candidates->len; i++)
{
const char *candidate = candidates->pdata[i];
guint64 candidate_ts;
g_autoptr(GVariant) commit = NULL;
OstreeRepoCommitState state;
gboolean have_candidate;
/* Do we have this commit at all? If not, skip it */
if (!ostree_repo_has_object (pull_data->repo, OSTREE_OBJECT_TYPE_COMMIT,
candidate, &have_candidate,
NULL, error))
return FALSE;
if (!have_candidate)
continue;
/* Load it */
if (!ostree_repo_load_commit (pull_data->repo, candidate,
&commit, &state, error))
return FALSE;
/* Ignore partial commits, we can't use them */
if (state & OSTREE_REPO_COMMIT_STATE_PARTIAL)
continue;
/* Is it newer? */
candidate_ts = ostree_commit_get_timestamp (commit);
if (newest_candidate == NULL ||
candidate_ts > newest_candidate_timestamp)
{
newest_candidate = candidate;
newest_candidate_timestamp = candidate_ts;
}
}
*out_from_revision = g_strdup (newest_candidate);
return TRUE;
}
typedef struct {
OtPullData *pull_data;
char *from_revision;
@@ -1926,7 +2188,7 @@ _ostree_repo_remote_new_fetcher (OstreeRepo *self,
if (tls_permissive)
fetcher_flags |= OSTREE_FETCHER_FLAGS_TLS_PERMISSIVE;
fetcher = _ostree_fetcher_new (self->tmp_dir_fd, fetcher_flags);
fetcher = _ostree_fetcher_new (self->tmp_dir_fd, remote_name, fetcher_flags);
{
g_autofree char *tls_client_cert_path = NULL;
@@ -2327,6 +2589,7 @@ reinitialize_fetcher (OtPullData *pull_data, const char *remote_name, GError **e
* * override-url (s): Fetch objects from this URL if remote specifies no metalink in options
* * inherit-transaction (b): Don't initiate, finish or abort a transaction, usefult to do mutliple pulls in one transaction.
* * http-headers (a(ss)): Additional headers to add to all HTTP requests
* * update-frequency (u): Frequency to call the async progress callback in milliseconds, if any; only values higher than 0 are valid
*/
gboolean
ostree_repo_pull_with_options (OstreeRepo *self,
@@ -2351,6 +2614,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
char **configured_branches = NULL;
guint64 bytes_transferred;
guint64 end_time;
guint update_frequency = 0;
OstreeRepoPullFlags flags = 0;
const char *dir_to_pull = NULL;
g_autofree char **dirs_to_pull = NULL;
@@ -2361,7 +2625,6 @@ ostree_repo_pull_with_options (OstreeRepo *self,
gboolean opt_gpg_verify_set = FALSE;
gboolean opt_gpg_verify_summary_set = FALSE;
const char *url_override = NULL;
gboolean mirroring_into_archive;
gboolean inherit_transaction = FALSE;
int i;
@@ -2387,6 +2650,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
(void) g_variant_lookup (options, "override-url", "&s", &url_override);
(void) g_variant_lookup (options, "inherit-transaction", "b", &inherit_transaction);
(void) g_variant_lookup (options, "http-headers", "@a(ss)", &pull_data->extra_headers);
(void) g_variant_lookup (options, "update-frequency", "u", &update_frequency);
}
g_return_val_if_fail (pull_data->maxdepth >= -1, FALSE);
@@ -2400,6 +2664,7 @@ ostree_repo_pull_with_options (OstreeRepo *self,
g_return_val_if_fail (dirs_to_pull[i][0] == '/', FALSE);
g_return_val_if_fail (!(disable_static_deltas && pull_data->require_static_deltas), FALSE);
/* We only do dry runs with static deltas, because we don't really have any
* in-advance information for bare fetches.
*/
@@ -2410,8 +2675,6 @@ ostree_repo_pull_with_options (OstreeRepo *self,
pull_data->is_untrusted = (flags & OSTREE_REPO_PULL_FLAGS_UNTRUSTED) > 0;
pull_data->cancellable = cancellable ? g_object_ref (cancellable) : NULL;
mirroring_into_archive = pull_data->is_mirror && self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2;
if (error)
pull_data->async_error = &pull_data->cached_async_error;
else
@@ -2435,8 +2698,18 @@ ostree_repo_pull_with_options (OstreeRepo *self,
(GDestroyNotify)g_variant_unref, NULL);
pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, NULL);
pull_data->requested_fallback_content = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free, NULL);
pull_data->requested_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify)g_variant_unref, NULL);
pull_data->pending_fetch_content = g_hash_table_new_full (g_str_hash, g_str_equal,
(GDestroyNotify)g_free,
(GDestroyNotify)fetch_object_data_free);
pull_data->pending_fetch_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal,
(GDestroyNotify)g_variant_unref,
(GDestroyNotify)fetch_object_data_free);
pull_data->pending_fetch_deltaparts = g_hash_table_new_full (NULL, NULL, (GDestroyNotify)fetch_static_delta_data_free, NULL);
if (dir_to_pull != NULL || dirs_to_pull != NULL)
{
pull_data->dirs = g_ptr_array_new_with_free_func (g_free);
@@ -2671,6 +2944,23 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (pull_data->remote_repo_local && !pull_data->require_static_deltas)
disable_static_deltas = TRUE;
/* We can't use static deltas if pulling into an archive-z2 repo. */
if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2)
{
if (pull_data->require_static_deltas)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Can't use static deltas in an archive repo");
goto out;
}
disable_static_deltas = TRUE;
}
/* It's not efficient to use static deltas if all we want is the commit
* metadata. */
if (pull_data->is_commit_only)
disable_static_deltas = TRUE;
pull_data->static_delta_superblocks = g_ptr_array_new_with_free_func ((GDestroyNotify)g_variant_unref);
{
@@ -2953,11 +3243,23 @@ ostree_repo_pull_with_options (OstreeRepo *self,
const char *ref = key;
const char *to_revision = value;
if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE,
&from_revision, error))
goto out;
/* If we have a summary, find the latest local commit we have
* to use as a from revision for static deltas.
*/
if (pull_data->summary)
{
if (!get_best_static_delta_start_for (pull_data, to_revision, &from_revision,
cancellable, error))
goto out;
}
else
{
if (!ostree_repo_resolve_rev (pull_data->repo, ref, TRUE,
&from_revision, error))
goto out;
}
if (!(disable_static_deltas || mirroring_into_archive || pull_data->is_commit_only) &&
if (!disable_static_deltas &&
(from_revision == NULL || g_strcmp0 (from_revision, to_revision) != 0))
{
g_autofree char *delta_name =
@@ -2984,7 +3286,12 @@ ostree_repo_pull_with_options (OstreeRepo *self,
if (pull_data->progress)
{
update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1);
/* Setup a custom frequency if set */
if (update_frequency > 0)
update_timeout = g_timeout_source_new (pull_data->dry_run ? 0 : update_frequency);
else
update_timeout = g_timeout_source_new_seconds (pull_data->dry_run ? 0 : 1);
g_source_set_priority (update_timeout, G_PRIORITY_HIGH);
g_source_set_callback (update_timeout, update_progress, pull_data, NULL);
g_source_attach (update_timeout, pull_data->main_context);
@@ -3147,7 +3454,11 @@ ostree_repo_pull_with_options (OstreeRepo *self,
g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->summary_deltas_checksums, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_fallback_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_content, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_metadata, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->pending_fetch_deltaparts, (GDestroyNotify) g_hash_table_unref);
g_clear_pointer (&pull_data->idle_src, (GDestroyNotify) g_source_destroy);
g_clear_pointer (&pull_data->dirs, (GDestroyNotify) g_ptr_array_unref);
g_clear_pointer (&remote_config, (GDestroyNotify) g_key_file_unref);
@@ -3242,7 +3553,7 @@ out:
return ret;
}
#else /* HAVE_LIBSOUP */
#else /* HAVE_LIBCURL_OR_LIBSOUP */
gboolean
ostree_repo_pull_with_options (OstreeRepo *self,
@@ -3271,4 +3582,4 @@ ostree_repo_remote_fetch_summary_with_options (OstreeRepo *self,
return FALSE;
}
#endif /* HAVE_LIBSOUP */
#endif /* HAVE_LIBCURL_OR_LIBSOUP */
src/libostree/ostree-repo-static-delta-compilation.c
+1 -5
View File
@@ -1003,11 +1003,7 @@ generate_delta_lowlatency (OstreeRepo *repo,
if (!check_object_world_readable (repo, from_checksum, &from_world_readable, cancellable, error))
goto out;
if (!from_world_readable)
{
g_hash_table_iter_steal (&hashiter);
g_hash_table_add (new_reachable_regfile_content, (char*)from_checksum);
continue;
}
continue;
if (!try_content_rollsum (repo, opts, from_checksum, to_checksum,
&rollsum, cancellable, error))
src/libostree/ostree-repo-static-delta-core.c
+5 -5
View File
@@ -884,10 +884,6 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
OT_VARIANT_MAP_TRUSTED, &delta_superblock, error))
goto out;
{ g_autofree char *variant_string = g_variant_print (delta_superblock, 1);
g_print ("%s\n", variant_string);
}
g_print ("Delta: %s\n", delta_id);
{ const char *endianness_description;
gboolean was_heuristic;
@@ -940,9 +936,13 @@ _ostree_repo_static_delta_dump (OstreeRepo *self,
for (i = 0; i < n_fallback; i++)
{
guint64 size, usize;
g_variant_get_child (fallback, i, "(y@aytt)", NULL, NULL, &size, &usize);
g_autoptr(GVariant) checksum_v = NULL;
char checksum[OSTREE_SHA256_STRING_LEN+1];
g_variant_get_child (fallback, i, "(y@aytt)", NULL, &checksum_v, &size, &usize);
ostree_checksum_inplace_from_bytes (ostree_checksum_bytes_peek (checksum_v), checksum);
size = maybe_swap_endian_u64 (swap_endian, size);
usize = maybe_swap_endian_u64 (swap_endian, usize);
g_print (" %s\n", checksum);
total_fallback_size += size;
total_fallback_usize += usize;
}
src/libostree/ostree-repo.c
+209 -129
View File
@@ -284,7 +284,7 @@ _ostree_repo_remote_name_is_file (const char *remote_name)
* OSTree remotes are represented by keyfile groups, formatted like:
* `[remote "remotename"]`. This function returns a value named @option_name
* underneath that group, or @default_value if the remote exists but not the
* option name.
* option name. If an error is returned, @out_value will be set to %NULL.
*
* Returns: %TRUE on success, otherwise %FALSE with @error set
*/
@@ -360,8 +360,9 @@ ostree_repo_get_remote_option (OstreeRepo *self,
*
* OSTree remotes are represented by keyfile groups, formatted like:
* `[remote "remotename"]`. This function returns a value named @option_name
* underneath that group, and returns it as an zero terminated array of strings.
* If the option is not set, @out_value will be set to %NULL.
* underneath that group, and returns it as a zero terminated array of strings.
* If the option is not set, or if an error is returned, @out_value will be set
* to %NULL.
*
* Returns: %TRUE on success, otherwise %FALSE with @error set
*/
@@ -435,7 +436,8 @@ ostree_repo_get_remote_list_option (OstreeRepo *self,
* OSTree remotes are represented by keyfile groups, formatted like:
* `[remote "remotename"]`. This function returns a value named @option_name
* underneath that group, and returns it as a boolean.
* If the option is not set, @out_value will be set to @default_value.
* If the option is not set, @out_value will be set to @default_value. If an
* error is returned, @out_value will be set to %FALSE.
*
* Returns: %TRUE on success, otherwise %FALSE with @error set
*/
@@ -2002,14 +2004,138 @@ get_remotes_d_dir (OstreeRepo *self)
}
static gboolean
append_remotes_d (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
reload_core_config (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
g_autofree char *version = NULL;
g_autofree char *mode = NULL;
g_autofree char *contents = NULL;
g_autofree char *parent_repo_path = NULL;
gboolean is_archive;
gsize len;
g_clear_pointer (&self->config, (GDestroyNotify)g_key_file_unref);
self->config = g_key_file_new ();
contents = glnx_file_get_contents_utf8_at (self->repo_dir_fd, "config", &len,
NULL, error);
if (!contents)
return FALSE;
if (!g_key_file_load_from_data (self->config, contents, len, 0, error))
{
g_prefix_error (error, "Couldn't parse config file: ");
return FALSE;
}
version = g_key_file_get_value (self->config, "core", "repo_version", error);
if (!version)
return FALSE;
if (strcmp (version, "1") != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid repository version '%s'", version);
return FALSE;
}
if (!ot_keyfile_get_boolean_with_default (self->config, "core", "archive",
FALSE, &is_archive, error))
return FALSE;
if (is_archive)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of OSTree no longer supports \"archive\" repositories; use archive-z2 instead");
return FALSE;
}
if (!ot_keyfile_get_value_with_default (self->config, "core", "mode",
"bare", &mode, error))
return FALSE;
if (!ostree_repo_mode_from_string (mode, &self->mode, error))
return FALSE;
if (self->writable)
{
if (!ot_keyfile_get_boolean_with_default (self->config, "core", "enable-uncompressed-cache",
TRUE, &self->enable_uncompressed_cache, error))
return FALSE;
}
else
self->enable_uncompressed_cache = FALSE;
{
gboolean do_fsync;
if (!ot_keyfile_get_boolean_with_default (self->config, "core", "fsync",
TRUE, &do_fsync, error))
return FALSE;
if (!do_fsync)
ostree_repo_set_disable_fsync (self, TRUE);
}
{ g_autofree char *tmp_expiry_seconds = NULL;
/* 86400 secs = one day */
if (!ot_keyfile_get_value_with_default (self->config, "core", "tmp-expiry-secs", "86400",
&tmp_expiry_seconds, error))
return FALSE;
self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10);
}
{ g_autofree char *compression_level_str = NULL;
/* gzip defaults to 6 */
(void)ot_keyfile_get_value_with_default (self->config, "archive", "zlib-level", NULL,
&compression_level_str, NULL);
if (compression_level_str)
/* Ensure level is in [1,9] */
self->zlib_compression_level = MAX (1, MIN (9, g_ascii_strtoull (compression_level_str, NULL, 10)));
else
self->zlib_compression_level = OSTREE_ARCHIVE_DEFAULT_COMPRESSION_LEVEL;
}
if (!ot_keyfile_get_value_with_default (self->config, "core", "parent",
NULL, &parent_repo_path, error))
return FALSE;
if (parent_repo_path && parent_repo_path[0])
{
g_autoptr(GFile) parent_repo_f = g_file_new_for_path (parent_repo_path);
g_clear_object (&self->parent_repo);
self->parent_repo = ostree_repo_new (parent_repo_f);
if (!ostree_repo_open (self->parent_repo, cancellable, error))
{
g_prefix_error (error, "While checking parent repository '%s': ",
gs_file_get_path_cached (parent_repo_f));
return FALSE;
}
}
return TRUE;
}
static gboolean
reload_remote_config (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
g_autoptr(GFile) remotes_d = NULL;
g_autoptr(GFileEnumerator) direnum = NULL;
g_mutex_lock (&self->remotes_lock);
g_hash_table_remove_all (self->remotes);
g_mutex_unlock (&self->remotes_lock);
if (!add_remotes_from_keyfile (self, self->config, NULL, error))
goto out;
remotes_d = get_remotes_d_dir (self);
if (remotes_d == NULL)
return TRUE;
@@ -2050,17 +2176,34 @@ append_remotes_d (OstreeRepo *self,
return ret;
}
/**
* ostree_repo_reload_config:
* @self: repo
* @cancellable: cancellable
* @error: error
*
* By default, an #OstreeRepo will cache the remote configuration and its
* own repo/config data. This API can be used to reload it.
*/
gboolean
ostree_repo_reload_config (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
if (!reload_core_config (self, cancellable, error))
return FALSE;
if (!reload_remote_config (self, cancellable, error))
return FALSE;
return TRUE;
}
gboolean
ostree_repo_open (OstreeRepo *self,
GCancellable *cancellable,
GError **error)
{
gboolean ret = FALSE;
gboolean is_archive;
struct stat stbuf;
g_autofree char *version = NULL;
g_autofree char *mode = NULL;
g_autofree char *parent_repo_path = NULL;
g_return_val_if_fail (error == NULL || *error == NULL, FALSE);
@@ -2126,102 +2269,6 @@ ostree_repo_open (OstreeRepo *self,
self->target_owner_uid = self->target_owner_gid = -1;
}
self->config = g_key_file_new ();
{ g_autofree char *contents = NULL;
gsize len;
contents = glnx_file_get_contents_utf8_at (self->repo_dir_fd, "config", &len,
NULL, error);
if (!contents)
goto out;
if (!g_key_file_load_from_data (self->config, contents, len, 0, error))
{
g_prefix_error (error, "Couldn't parse config file: ");
goto out;
}
}
if (!add_remotes_from_keyfile (self, self->config, NULL, error))
goto out;
version = g_key_file_get_value (self->config, "core", "repo_version", error);
if (!version)
goto out;
if (strcmp (version, "1") != 0)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
"Invalid repository version '%s'", version);
goto out;
}
if (!ot_keyfile_get_boolean_with_default (self->config, "core", "archive",
FALSE, &is_archive, error))
goto out;
if (is_archive)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
"This version of OSTree no longer supports \"archive\" repositories; use archive-z2 instead");
goto out;
}
if (!ot_keyfile_get_value_with_default (self->config, "core", "mode",
"bare", &mode, error))
goto out;
if (!ostree_repo_mode_from_string (mode, &self->mode, error))
goto out;
if (!ot_keyfile_get_value_with_default (self->config, "core", "parent",
NULL, &parent_repo_path, error))
goto out;
if (parent_repo_path && parent_repo_path[0])
{
g_autoptr(GFile) parent_repo_f = g_file_new_for_path (parent_repo_path);
self->parent_repo = ostree_repo_new (parent_repo_f);
if (!ostree_repo_open (self->parent_repo, cancellable, error))
{
g_prefix_error (error, "While checking parent repository '%s': ",
gs_file_get_path_cached (parent_repo_f));
goto out;
}
}
if (self->writable)
{
if (!ot_keyfile_get_boolean_with_default (self->config, "core", "enable-uncompressed-cache",
TRUE, &self->enable_uncompressed_cache, error))
goto out;
}
else
self->enable_uncompressed_cache = FALSE;
{
gboolean do_fsync;
if (!ot_keyfile_get_boolean_with_default (self->config, "core", "fsync",
TRUE, &do_fsync, error))
goto out;
if (!do_fsync)
ostree_repo_set_disable_fsync (self, TRUE);
}
{ g_autofree char *tmp_expiry_seconds = NULL;
/* 86400 secs = one day */
if (!ot_keyfile_get_value_with_default (self->config, "core", "tmp-expiry-secs", "86400",
&tmp_expiry_seconds, error))
goto out;
self->tmp_expiry_seconds = g_ascii_strtoull (tmp_expiry_seconds, NULL, 10);
}
if (!append_remotes_d (self, cancellable, error))
goto out;
if (!glnx_opendirat (self->repo_dir_fd, "tmp", TRUE, &self->tmp_dir_fd, error))
goto out;
@@ -2234,6 +2281,10 @@ ostree_repo_open (OstreeRepo *self,
goto out;
}
if (!ostree_repo_reload_config (self, cancellable, error))
goto out;
/* TODO - delete this */
if (self->mode == OSTREE_REPO_MODE_ARCHIVE_Z2 && self->enable_uncompressed_cache)
{
if (!glnx_shutil_mkdir_p_at (self->repo_dir_fd, "uncompressed-objects-cache", 0755,
@@ -3797,8 +3848,7 @@ _formatted_time_remaining_from_seconds (guint64 seconds_remaining)
if (minutes_remaining)
g_string_append_printf (description, "%" G_GUINT64_FORMAT " minutes ", minutes_remaining % 60);
if (seconds_remaining)
g_string_append_printf (description, "%" G_GUINT64_FORMAT " seconds ", seconds_remaining % 60);
g_string_append_printf (description, "%" G_GUINT64_FORMAT " seconds ", seconds_remaining % 60);
return g_string_free (description, FALSE);
}
@@ -3827,12 +3877,15 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress
{
GString *buf;
g_autofree char *status = NULL;
gboolean scanning;
guint outstanding_fetches;
guint outstanding_metadata_fetches;
guint outstanding_writes;
guint n_scanned_metadata;
guint fetched_delta_parts;
guint total_delta_parts;
guint fetched_delta_part_fallbacks;
guint total_delta_part_fallbacks;
buf = g_string_new ("");
@@ -3840,9 +3893,12 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress
outstanding_fetches = ostree_async_progress_get_uint (progress, "outstanding-fetches");
outstanding_metadata_fetches = ostree_async_progress_get_uint (progress, "outstanding-metadata-fetches");
outstanding_writes = ostree_async_progress_get_uint (progress, "outstanding-writes");
scanning = ostree_async_progress_get_uint (progress, "scanning") == 1;
n_scanned_metadata = ostree_async_progress_get_uint (progress, "scanned-metadata");
fetched_delta_parts = ostree_async_progress_get_uint (progress, "fetched-delta-parts");
total_delta_parts = ostree_async_progress_get_uint (progress, "total-delta-parts");
fetched_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "fetched-delta-fallbacks");
total_delta_part_fallbacks = ostree_async_progress_get_uint (progress, "total-delta-fallbacks");
if (status)
{
@@ -3860,35 +3916,56 @@ ostree_repo_pull_default_console_progress_changed (OstreeAsyncProgress *progress
g_autofree char *formatted_bytes_transferred =
g_format_size_full (bytes_transferred, 0);
g_autofree char *formatted_bytes_sec = NULL;
g_autofree char *formatted_est_time_remaining = NULL;
guint64 bytes_sec;
/* Ignore the first second, or when we haven't transferred any
* data, since those could cause divide by zero below.
*/
if ((current_time - start_time) < G_USEC_PER_SEC || bytes_transferred == 0)
{
bytes_sec = 0;
formatted_bytes_sec = g_strdup ("-");
formatted_est_time_remaining = g_strdup ("- ");
}
else
{
guint64 bytes_sec = bytes_transferred / ((current_time - start_time) / G_USEC_PER_SEC);
guint64 est_time_remaining = (total_delta_part_size - bytes_transferred) / bytes_sec;
bytes_sec = bytes_transferred / ((current_time - start_time) / G_USEC_PER_SEC);
formatted_bytes_sec = g_format_size (bytes_sec);
formatted_est_time_remaining = _formatted_time_remaining_from_seconds (est_time_remaining);
}
/* Are we doing deltas? If so, we can be more accurate */
if (total_delta_parts > 0)
{
g_autofree char *formatted_total =
g_format_size (total_delta_part_size);
/* No space between %s and remaining, since formatted_est_time_remaining has a trailing space */
g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/s %s/%s %sremaining",
fetched_delta_parts, total_delta_parts,
formatted_bytes_sec, formatted_bytes_transferred,
formatted_total, formatted_est_time_remaining);
guint64 fetched_delta_part_size = ostree_async_progress_get_uint64 (progress, "fetched-delta-part-size");
g_autofree char *formatted_fetched = NULL;
g_autofree char *formatted_total = NULL;
/* Here we merge together deltaparts + fallbacks to avoid bloating the text UI */
fetched_delta_parts += fetched_delta_part_fallbacks;
total_delta_parts += total_delta_part_fallbacks;
formatted_fetched = g_format_size (fetched_delta_part_size);
formatted_total = g_format_size (total_delta_part_size);
if (bytes_sec > 0)
{
/* MAX(0, value) here just to be defensive */
guint64 est_time_remaining = MAX(0, (total_delta_part_size - fetched_delta_part_size)) / bytes_sec;
g_autofree char *formatted_est_time_remaining = _formatted_time_remaining_from_seconds (est_time_remaining);
/* No space between %s and remaining, since formatted_est_time_remaining has a trailing space */
g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/%s %s/s %sremaining",
fetched_delta_parts, total_delta_parts,
formatted_fetched, formatted_total,
formatted_bytes_sec,
formatted_est_time_remaining);
}
else
{
g_string_append_printf (buf, "Receiving delta parts: %u/%u %s/%s",
fetched_delta_parts, total_delta_parts,
formatted_fetched, formatted_total);
}
}
else if (outstanding_metadata_fetches)
else if (scanning || outstanding_metadata_fetches)
{
g_string_append_printf (buf, "Receiving metadata objects: %u/(estimating) %s/s %s",
metadata_fetched, formatted_bytes_sec, formatted_bytes_transferred);
@@ -4232,16 +4309,14 @@ find_keyring (OstreeRepo *self,
OstreeRemote *remote,
GCancellable *cancellable)
{
g_autoptr(GFile) remotes_d = NULL;
g_autoptr(GFile) file = NULL;
file = g_file_get_child (self->repodir, remote->keyring);
g_autoptr(GFile) file = g_file_get_child (self->repodir, remote->keyring);
if (g_file_query_exists (file, cancellable))
{
return g_steal_pointer (&file);
}
remotes_d = get_remotes_d_dir (self);
g_autoptr(GFile) remotes_d = get_remotes_d_dir (self);
if (remotes_d)
{
g_autoptr(GFile) file2 = g_file_get_child (remotes_d, remote->keyring);
@@ -4356,8 +4431,8 @@ _ostree_repo_gpg_verify_with_metadata (OstreeRepo *self,
_OSTREE_METADATA_GPGSIGS_TYPE);
if (!signaturedata)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"GPG verification enabled, but no signatures found (use gpg-verify=false in remote config to disable)");
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND,
"GPG verification enabled, but no signatures found (use gpg-verify=false in remote config to disable)");
return NULL;
}
@@ -4474,7 +4549,12 @@ ostree_repo_verify_commit (OstreeRepo *self,
keyringdir, extra_keyring,
cancellable, error);
return ostree_gpg_verify_result_require_valid_signature (result, error);
if (!ostree_gpg_verify_result_require_valid_signature (result, error))
{
g_prefix_error (error, "Commit %s: ", commit_checksum);
return FALSE;
}
return TRUE;
}
/**
src/libostree/ostree-repo.h
+9 -2
View File
@@ -101,6 +101,11 @@ GKeyFile * ostree_repo_get_config (OstreeRepo *self);
_OSTREE_PUBLIC
GKeyFile * ostree_repo_copy_config (OstreeRepo *self);
_OSTREE_PUBLIC
gboolean ostree_repo_reload_config (OstreeRepo *self,
GCancellable *cancellable,
GError **error);
_OSTREE_PUBLIC
gboolean ostree_repo_remote_add (OstreeRepo *self,
const char *name,
@@ -717,11 +722,13 @@ typedef enum {
/**
* OstreeRepoCheckoutOverwriteMode:
* @OSTREE_REPO_CHECKOUT_OVERWRITE_NONE: No special options
* @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, overwrite earlier files, but keep earlier directories
* @OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES: When layering checkouts, unlink() and replace existing files, but do not modify existing directories
* @OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES: Only add new files/directories
*/
typedef enum {
OSTREE_REPO_CHECKOUT_OVERWRITE_NONE = 0,
OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1
OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES = 1,
OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES = 2, /* Since: 2017.3 */
} OstreeRepoCheckoutOverwriteMode;
_OSTREE_PUBLIC
src/libostree/ostree-soup-form.c
+140
View File
@@ -0,0 +1,140 @@
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
/* soup-form.c : utility functions for HTML forms */
/*
* Copyright 2008 Red Hat, Inc.
*/
/* This one is stripped down to only have soup_form_encode_hash()
* and soup_form_encode_valist() which are the only bits that soup-uri.c
* calls.
*/
#include <config.h>
#include <string.h>
#include "ostree-soup-uri.h"
/**
* SECTION:soup-form
* @short_description: HTML form handling
* @see_also: #SoupMultipart
*
* libsoup contains several help methods for processing HTML forms as
* defined by <ulink
* url="http://www.w3.org/TR/html401/interact/forms.html#h-17.13">the
* HTML 4.01 specification</ulink>.
**/
/**
* SOUP_FORM_MIME_TYPE_URLENCODED:
*
* A macro containing the value
* <literal>"application/x-www-form-urlencoded"</literal>; the default
* MIME type for POSTing HTML form data.
*
* Since: 2.26
**/
/**
* SOUP_FORM_MIME_TYPE_MULTIPART:
*
* A macro containing the value
* <literal>"multipart/form-data"</literal>; the MIME type used for
* posting form data that contains files to be uploaded.
*
* Since: 2.26
**/
#define XDIGIT(c) ((c) <= '9' ? (c) - '0' : ((c) & 0x4F) - 'A' + 10)
#define HEXCHAR(s) ((XDIGIT (s[1]) << 4) + XDIGIT (s[2]))
static void
append_form_encoded (GString *str, const char *in)
{
const unsigned char *s = (const unsigned char *)in;
while (*s) {
if (*s == ' ') {
g_string_append_c (str, '+');
s++;
} else if (!g_ascii_isalnum (*s) && (*s != '-') && (*s != '_')
&& (*s != '.'))
g_string_append_printf (str, "%%%02X", (int)*s++);
else
g_string_append_c (str, *s++);
}
}
static void
encode_pair (GString *str, const char *name, const char *value)
{
g_return_if_fail (name != NULL);
g_return_if_fail (value != NULL);
if (str->len)
g_string_append_c (str, '&');
append_form_encoded (str, name);
g_string_append_c (str, '=');
append_form_encoded (str, value);
}
/**
* soup_form_encode_hash:
* @form_data_set: (element-type utf8 utf8): a hash table containing
* name/value pairs (as strings)
*
* Encodes @form_data_set into a value of type
* "application/x-www-form-urlencoded", as defined in the HTML 4.01
* spec.
*
* Note that the HTML spec states that "The control names/values are
* listed in the order they appear in the document." Since this method
* takes a hash table, it cannot enforce that; if you care about the
* ordering of the form fields, use soup_form_encode_datalist().
*
* Return value: the encoded form
**/
char *
soup_form_encode_hash (GHashTable *form_data_set)
{
GString *str = g_string_new (NULL);
GHashTableIter iter;
gpointer name, value;
g_hash_table_iter_init (&iter, form_data_set);
while (g_hash_table_iter_next (&iter, &name, &value))
encode_pair (str, name, value);
return g_string_free (str, FALSE);
}
/**
* soup_form_encode_valist:
* @first_field: name of the first form field
* @args: pointer to additional values, as in soup_form_encode()
*
* See soup_form_encode(). This is mostly an internal method, used by
* various other methods such as soup_uri_set_query_from_fields() and
* soup_form_request_new().
*
* Return value: the encoded form
**/
char *
soup_form_encode_valist (const char *first_field, va_list args)
{
GString *str = g_string_new (NULL);
const char *name, *value;
name = first_field;
value = va_arg (args, const char *);
while (name && value) {
encode_pair (str, name, value);
name = va_arg (args, const char *);
if (name)
value = va_arg (args, const char *);
}
return g_string_free (str, FALSE);
}
src/libostree/ostree-soup-uri.c
+1483
View File
File diff suppressed because it is too large. Load diff
src/libostree/ostree-soup-uri.h
+147
View File
@@ -0,0 +1,147 @@
/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 8 -*- */
/*
* Copyright 1999-2002 Ximian, Inc.
*/
/* NOTE - taken from the libsoup codebase for use by the ostree curl backend
* (yes, ironically enough).
*
* Please watch for future changes in libsoup.
*/
#ifndef SOUP_URI_H
#define SOUP_URI_H 1
/* OSTREECHANGE: make struct private
* Only include gio, and skip available definitions.
*/
#include <gio/gio.h>
#define SOUP_AVAILABLE_IN_2_4
#define SOUP_AVAILABLE_IN_2_28
#define SOUP_AVAILABLE_IN_2_32
G_BEGIN_DECLS
/* OSTREECHANGE: make struct private */
typedef struct _SoupURI SoupURI;
/* OSTREECHANGE: import soup-misc's interning */
#define SOUP_VAR extern
#define _SOUP_ATOMIC_INTERN_STRING(variable, value) ((const char *)(g_atomic_pointer_get (&(variable)) ? (variable) : (g_atomic_pointer_set (&(variable), (gpointer)g_intern_static_string (value)), (variable))))
#define SOUP_URI_SCHEME_HTTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTP, "http")
#define SOUP_URI_SCHEME_HTTPS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_HTTPS, "https")
#define SOUP_URI_SCHEME_FTP _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FTP, "ftp")
#define SOUP_URI_SCHEME_FILE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_FILE, "file")
#define SOUP_URI_SCHEME_DATA _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_DATA, "data")
#define SOUP_URI_SCHEME_RESOURCE _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_RESOURCE, "resource")
#define SOUP_URI_SCHEME_WS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WS, "ws")
#define SOUP_URI_SCHEME_WSS _SOUP_ATOMIC_INTERN_STRING (_SOUP_URI_SCHEME_WSS, "wss")
/* OSTREECHANGE: import soup-form bits */
SOUP_AVAILABLE_IN_2_4
char *soup_form_encode_hash (GHashTable *form_data_set);
SOUP_AVAILABLE_IN_2_4
char *soup_form_encode_valist (const char *first_field,
va_list args);
SOUP_VAR gpointer _SOUP_URI_SCHEME_HTTP, _SOUP_URI_SCHEME_HTTPS;
SOUP_VAR gpointer _SOUP_URI_SCHEME_FTP;
SOUP_VAR gpointer _SOUP_URI_SCHEME_FILE, _SOUP_URI_SCHEME_DATA, _SOUP_URI_SCHEME_RESOURCE;
SOUP_VAR gpointer _SOUP_URI_SCHEME_WS, _SOUP_URI_SCHEME_WSS;
SOUP_AVAILABLE_IN_2_4
SoupURI *soup_uri_new_with_base (SoupURI *base,
const char *uri_string);
SOUP_AVAILABLE_IN_2_4
SoupURI *soup_uri_new (const char *uri_string);
SOUP_AVAILABLE_IN_2_4
char *soup_uri_to_string (SoupURI *uri,
gboolean just_path_and_query);
SOUP_AVAILABLE_IN_2_4
SoupURI *soup_uri_copy (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
gboolean soup_uri_equal (SoupURI *uri1,
SoupURI *uri2);
SOUP_AVAILABLE_IN_2_4
void soup_uri_free (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
char *soup_uri_encode (const char *part,
const char *escape_extra);
SOUP_AVAILABLE_IN_2_4
char *soup_uri_decode (const char *part);
SOUP_AVAILABLE_IN_2_4
char *soup_uri_normalize (const char *part,
const char *unescape_extra);
SOUP_AVAILABLE_IN_2_4
gboolean soup_uri_uses_default_port (SoupURI *uri);
SOUP_AVAILABLE_IN_2_32
const char *soup_uri_get_scheme (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_scheme (SoupURI *uri,
const char *scheme);
SOUP_AVAILABLE_IN_2_32
const char *soup_uri_get_user (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_user (SoupURI *uri,
const char *user);
SOUP_AVAILABLE_IN_2_32
const char *soup_uri_get_password (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_password (SoupURI *uri,
const char *password);
SOUP_AVAILABLE_IN_2_32
const char *soup_uri_get_host (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_host (SoupURI *uri,
const char *host);
SOUP_AVAILABLE_IN_2_32
guint soup_uri_get_port (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_port (SoupURI *uri,
guint port);
SOUP_AVAILABLE_IN_2_32
const char *soup_uri_get_path (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_path (SoupURI *uri,
const char *path);
SOUP_AVAILABLE_IN_2_32
const char *soup_uri_get_query (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_query (SoupURI *uri,
const char *query);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_query_from_form (SoupURI *uri,
GHashTable *form);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_query_from_fields (SoupURI *uri,
const char *first_field,
...) G_GNUC_NULL_TERMINATED;
SOUP_AVAILABLE_IN_2_32
const char *soup_uri_get_fragment (SoupURI *uri);
SOUP_AVAILABLE_IN_2_4
void soup_uri_set_fragment (SoupURI *uri,
const char *fragment);
SOUP_AVAILABLE_IN_2_28
SoupURI *soup_uri_copy_host (SoupURI *uri);
SOUP_AVAILABLE_IN_2_28
guint soup_uri_host_hash (gconstpointer key);
SOUP_AVAILABLE_IN_2_28
gboolean soup_uri_host_equal (gconstpointer v1,
gconstpointer v2);
#define SOUP_URI_IS_VALID(uri) ((uri) && (uri)->scheme && (uri)->path)
#define SOUP_URI_VALID_FOR_HTTP(uri) ((uri) && ((uri)->scheme == SOUP_URI_SCHEME_HTTP || (uri)->scheme == SOUP_URI_SCHEME_HTTPS) && (uri)->host && (uri)->path)
G_END_DECLS
#endif /*SOUP_URI_H*/
src/libostree/ostree-sysroot-deploy.c
+6 -1
View File
@@ -1692,8 +1692,13 @@ is_ro_mount (const char *path)
fs = mnt_table_find_target(tb, path, MNT_ITER_BACKWARD);
is_mount = fs && mnt_fs_get_target (fs);
mnt_free_cache (cache);
#ifdef HAVE_MNT_UNREF_CACHE
mnt_unref_table (tb);
mnt_unref_cache (cache);
#else
mnt_free_table (tb);
mnt_free_cache (cache);
#endif
if (!is_mount)
return FALSE;
src/libostree/ostree-sysroot-upgrader.c
+2 -1
View File
@@ -552,7 +552,8 @@ ostree_sysroot_upgrader_pull_one_dir (OstreeSysrootUpgrader *self,
g_assert (self->merge_deployment);
from_revision = ostree_deployment_get_csum (self->merge_deployment);
if (self->origin_remote)
if (self->origin_remote &&
(upgrader_flags & OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_SYNTHETIC) == 0)
{
if (!ostree_repo_pull_one_dir (repo, self->origin_remote, dir_to_pull, refs_to_fetch,
flags, progress,
src/libostree/ostree-sysroot-upgrader.h
+2 -1
View File
@@ -85,7 +85,8 @@ gboolean ostree_sysroot_upgrader_check_timestamps (OstreeRepo *repo,
typedef enum {
OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_NONE = 0,
OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER = (1 << 0)
OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_ALLOW_OLDER = (1 << 0),
OSTREE_SYSROOT_UPGRADER_PULL_FLAGS_SYNTHETIC = (1 << 1) /* Don't actually do a pull, just check timestamps/changed */
} OstreeSysrootUpgraderPullFlags;
_OSTREE_PUBLIC