From f8f5da219edd2279322bba916879fd53c2b65350 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 31 Jul 2014 18:50:19 -0400 Subject: [PATCH] Add repository "summary" file and metalink support For Fedora and potentially other distributions which use globally distributed mirrors, metalink is a popular solution to redirect clients to a dynamic set of mirrors. In order to make metalink work though, it needs *one* file which can be checksummed. (Well, potentially we could explode all refs into the metalink.xml, but that would be a lot more invasive, and a bit weird as we'd end up checksumming the checksum file). This commit adds a new command: $ ostree summary -u To regenerate the summary file. Can only be run by one process at a time. After that's done, the metalink can be generated based on it, and the client fetch code will parse and load it. https://bugzilla.gnome.org/show_bug.cgi?id=729585 --- Makefile-libostree.am | 2 + Makefile-ostree.am | 1 + Makefile-tests.am | 1 + src/libostree/ostree-core.h | 9 + src/libostree/ostree-metalink.c | 710 ++++++++++++++++++++++++++++++ src/libostree/ostree-metalink.h | 66 +++ src/libostree/ostree-repo-pull.c | 211 +++++++-- src/libostree/ostree-repo.c | 86 +++- src/libostree/ostree-repo.h | 6 + src/libotutil/ot-checksum-utils.c | 26 ++ src/libotutil/ot-checksum-utils.h | 5 + src/libotutil/ot-variant-utils.c | 64 +++ src/libotutil/ot-variant-utils.h | 7 + src/ostree/main.c | 1 + src/ostree/ot-builtin-summary.c | 63 +++ src/ostree/ot-builtins.h | 1 + tests/test-pull-metalink.sh | 120 +++++ 17 files changed, 1352 insertions(+), 27 deletions(-) create mode 100644 src/libostree/ostree-metalink.c create mode 100644 src/libostree/ostree-metalink.h create mode 100644 src/ostree/ot-builtin-summary.c create mode 100755 tests/test-pull-metalink.sh diff --git a/Makefile-libostree.am b/Makefile-libostree.am index 1b756001..e7b141ae 100644 --- a/Makefile-libostree.am +++ b/Makefile-libostree.am @@ -112,6 +112,8 @@ if USE_LIBSOUP libostree_1_la_SOURCES += \ src/libostree/ostree-fetcher.h \ src/libostree/ostree-fetcher.c \ + src/libostree/ostree-metalink.h \ + src/libostree/ostree-metalink.c \ src/libostree/ostree-repo-pull.c \ $(NULL) libostree_1_la_CFLAGS += $(OT_INTERNAL_SOUP_CFLAGS) diff --git a/Makefile-ostree.am b/Makefile-ostree.am index f1381acd..76df368b 100644 --- a/Makefile-ostree.am +++ b/Makefile-ostree.am @@ -40,6 +40,7 @@ ostree_SOURCES = src/ostree/main.c \ src/ostree/ot-builtin-remote.c \ src/ostree/ot-builtin-reset.c \ src/ostree/ot-builtin-rev-parse.c \ + src/ostree/ot-builtin-summary.c \ src/ostree/ot-builtin-show.c \ src/ostree/ot-builtin-static-delta.c \ src/ostree/ot-main.h \ diff --git a/Makefile-tests.am b/Makefile-tests.am index c36b1b84..03a2ff6a 100644 --- a/Makefile-tests.am +++ b/Makefile-tests.am @@ -31,6 +31,7 @@ testfiles = test-basic \ test-pull-archive-z \ test-pull-corruption \ test-pull-large-metadata \ + test-pull-metalink \ test-pull-resume \ test-gpg-signed-commit \ test-admin-deploy-syslinux \ diff --git a/src/libostree/ostree-core.h b/src/libostree/ostree-core.h index a867dbe4..e54e45d2 100644 --- a/src/libostree/ostree-core.h +++ b/src/libostree/ostree-core.h @@ -117,6 +117,15 @@ typedef enum { #define OSTREE_COMMIT_GVARIANT_STRING "(a{sv}aya(say)sstayay)" #define OSTREE_COMMIT_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_COMMIT_GVARIANT_STRING) +/** + * OSTREE_SUMMARY_GVARIANT_FORMAT: + * + * refs: a(s(taya{sv})) - Map of ref name -> (latest commit size, latest commit checksum, additional metadata), sorted by ref name + * extensions: a{sv} - Additional metadata, none defined at the current time + */ +#define OSTREE_SUMMARY_GVARIANT_STRING "(a(s(taya{sv}))a{sv})" +#define OSTREE_SUMMARY_GVARIANT_FORMAT G_VARIANT_TYPE (OSTREE_SUMMARY_GVARIANT_STRING) + /** * OstreeRepoMode: * @OSTREE_REPO_MODE_BARE: Files are stored as themselves; can only be written as root diff --git a/src/libostree/ostree-metalink.c b/src/libostree/ostree-metalink.c new file mode 100644 index 00000000..00cfb2c8 --- /dev/null +++ b/src/libostree/ostree-metalink.c @@ -0,0 +1,710 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2014 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ostree-metalink.h" + +#include "otutil.h" +#include "libgsystem.h" + +typedef enum { + OSTREE_METALINK_STATE_INITIAL, + OSTREE_METALINK_STATE_METALINK, + OSTREE_METALINK_STATE_FILES, + OSTREE_METALINK_STATE_FILE, + OSTREE_METALINK_STATE_SIZE, + OSTREE_METALINK_STATE_VERIFICATION, + OSTREE_METALINK_STATE_HASH, + OSTREE_METALINK_STATE_RESOURCES, + OSTREE_METALINK_STATE_URL, + + OSTREE_METALINK_STATE_PASSTHROUGH /* Ignoring unknown elements */ +} OstreeMetalinkState; + +struct OstreeMetalink +{ + GObject parent_instance; + + SoupURI *uri; + + OstreeFetcher *fetcher; + char *requested_file; + guint64 max_size; +}; + +G_DEFINE_TYPE (OstreeMetalink, _ostree_metalink, G_TYPE_OBJECT) + +typedef struct +{ + OstreeMetalink *metalink; + + GTask *task; + GMarkupParseContext *parser; + + guint passthrough_depth; + OstreeMetalinkState passthrough_previous; + + guint found_a_file_element : 1; + guint found_our_file_element : 1; + guint verification_known : 1; + + GChecksumType in_verification_type; + + guint64 size; + char *verification_sha256; + char *verification_sha512; + + GFile *result; + + char *last_metalink_error; + guint current_url_index; + GPtrArray *urls; + + OstreeMetalinkState state; +} OstreeMetalinkRequest; + +static void +state_transition (OstreeMetalinkRequest *self, + OstreeMetalinkState new_state) +{ + g_assert (self->state != new_state); + self->state = new_state; +} + +static void +unknown_element (OstreeMetalinkRequest *self, + const char *element_name, + GError **error) +{ + state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH); + g_assert (self->passthrough_depth == 0); +} + +static void +metalink_parser_start (GMarkupParseContext *context, + const gchar *element_name, + const gchar **attribute_names, + const gchar **attribute_values, + gpointer user_data, + GError **error) +{ + GTask *task = user_data; + OstreeMetalinkRequest *self = g_task_get_task_data (task); + + switch (self->state) + { + case OSTREE_METALINK_STATE_INITIAL: + if (strcmp (element_name, "metalink") == 0) + state_transition (self, OSTREE_METALINK_STATE_METALINK); + else + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_METALINK: + if (strcmp (element_name, "files") == 0) + state_transition (self, OSTREE_METALINK_STATE_FILES); + else + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_FILES: + /* If we've already processed a element we're OK with, just + * ignore the others. + */ + if (self->urls->len > 0) + { + state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH); + } + else if (strcmp (element_name, "file") == 0) + { + const char *file_name; + + if (!g_markup_collect_attributes (element_name, + attribute_names, + attribute_values, + error, + G_MARKUP_COLLECT_STRING, + "name", + &file_name, + G_MARKUP_COLLECT_INVALID)) + goto out; + + self->found_a_file_element = TRUE; + + if (strcmp (file_name, self->metalink->requested_file) != 0) + { + state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH); + g_assert (self->passthrough_depth == 0); + } + else + { + self->found_our_file_element = TRUE; + state_transition (self, OSTREE_METALINK_STATE_FILE); + } + } + else + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_FILE: + if (strcmp (element_name, "size") == 0) + state_transition (self, OSTREE_METALINK_STATE_SIZE); + else if (strcmp (element_name, "verification") == 0) + state_transition (self, OSTREE_METALINK_STATE_VERIFICATION); + else if (strcmp (element_name, "resources") == 0) + state_transition (self, OSTREE_METALINK_STATE_RESOURCES); + else + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_SIZE: + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_VERIFICATION: + if (strcmp (element_name, "hash") == 0) + { + char *verification_type_str = NULL; + + state_transition (self, OSTREE_METALINK_STATE_HASH); + if (!g_markup_collect_attributes (element_name, + attribute_names, + attribute_values, + error, + G_MARKUP_COLLECT_STRING, + "type", + &verification_type_str, + G_MARKUP_COLLECT_INVALID)) + goto out; + + /* Only accept sha256/sha512 */ + self->verification_known = TRUE; + if (strcmp (verification_type_str, "sha256") == 0) + self->in_verification_type = G_CHECKSUM_SHA256; + else if (strcmp (verification_type_str, "sha512") == 0) + self->in_verification_type = G_CHECKSUM_SHA512; + else + self->verification_known = FALSE; + } + else + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_HASH: + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_RESOURCES: + if (self->size == 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No element found or it is zero"); + goto out; + } + if (!self->verification_known) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No element with known found"); + goto out; + } + + if (strcmp (element_name, "url") == 0) + { + const char *protocol; + + if (!g_markup_collect_attributes (element_name, + attribute_names, + attribute_values, + error, + G_MARKUP_COLLECT_STRING, + "protocol", + &protocol, + G_MARKUP_COLLECT_STRING, + "type", + NULL, + G_MARKUP_COLLECT_STRING, + "location", + NULL, + G_MARKUP_COLLECT_STRING, + "preference", + NULL, + G_MARKUP_COLLECT_INVALID)) + goto out; + + /* Ignore non-HTTP resources */ + if (!(strcmp (protocol, "http") == 0 || strcmp (protocol, "https") == 0)) + state_transition (self, OSTREE_METALINK_STATE_PASSTHROUGH); + else + state_transition (self, OSTREE_METALINK_STATE_URL); + } + else + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_URL: + unknown_element (self, element_name, error); + break; + case OSTREE_METALINK_STATE_PASSTHROUGH: + self->passthrough_depth++; + break; + } + + out: + return; +} + +static void +metalink_parser_end (GMarkupParseContext *context, + const gchar *element_name, + gpointer user_data, + GError **error) +{ + GTask *task = user_data; + OstreeMetalinkRequest *self = g_task_get_task_data (task); + + switch (self->state) + { + case OSTREE_METALINK_STATE_INITIAL: + break; + case OSTREE_METALINK_STATE_METALINK: + state_transition (self, OSTREE_METALINK_STATE_INITIAL); + break; + case OSTREE_METALINK_STATE_FILES: + state_transition (self, OSTREE_METALINK_STATE_METALINK); + break; + case OSTREE_METALINK_STATE_FILE: + state_transition (self, OSTREE_METALINK_STATE_FILES); + break; + case OSTREE_METALINK_STATE_SIZE: + case OSTREE_METALINK_STATE_VERIFICATION: + case OSTREE_METALINK_STATE_RESOURCES: + state_transition (self, OSTREE_METALINK_STATE_FILE); + break; + case OSTREE_METALINK_STATE_HASH: + state_transition (self, OSTREE_METALINK_STATE_VERIFICATION); + break; + case OSTREE_METALINK_STATE_URL: + state_transition (self, OSTREE_METALINK_STATE_RESOURCES); + break; + case OSTREE_METALINK_STATE_PASSTHROUGH: + g_assert_cmpint (self->passthrough_depth, >, 0); + self->passthrough_depth--; + if (self->passthrough_depth == 0) + state_transition (self, self->passthrough_previous); + break; + } +} + +static void +metalink_parser_text (GMarkupParseContext *context, + const gchar *text, + gsize text_len, + gpointer user_data, + GError **error) +{ + GTask *task = user_data; + OstreeMetalinkRequest *self = g_task_get_task_data (task); + + switch (self->state) + { + case OSTREE_METALINK_STATE_INITIAL: + break; + case OSTREE_METALINK_STATE_METALINK: + break; + case OSTREE_METALINK_STATE_FILES: + break; + case OSTREE_METALINK_STATE_FILE: + break; + case OSTREE_METALINK_STATE_SIZE: + { + gs_free char *duped = g_strndup (text, text_len); + self->size = g_ascii_strtoull (duped, NULL, 10); + } + break; + case OSTREE_METALINK_STATE_VERIFICATION: + break; + case OSTREE_METALINK_STATE_HASH: + if (self->verification_known) + { + switch (self->in_verification_type) + { + case G_CHECKSUM_SHA256: + g_free (self->verification_sha256); + self->verification_sha256 = g_strndup (text, text_len); + break; + case G_CHECKSUM_SHA512: + g_free (self->verification_sha512); + self->verification_sha512 = g_strndup (text, text_len); + break; + default: + g_assert_not_reached (); + } + } + break; + case OSTREE_METALINK_STATE_RESOURCES: + break; + case OSTREE_METALINK_STATE_URL: + { + gs_free char *uri_text = g_strndup (text, text_len); + SoupURI *uri = soup_uri_new (uri_text); + if (uri != NULL) + g_ptr_array_add (self->urls, uri); + } + break; + case OSTREE_METALINK_STATE_PASSTHROUGH: + break; + } + +} + +static void +_ostree_metalink_finalize (GObject *object) +{ + OstreeMetalink *self; + + self = OSTREE_METALINK (object); + + g_object_unref (self->fetcher); + g_free (self->requested_file); + soup_uri_free (self->uri); + + G_OBJECT_CLASS (_ostree_metalink_parent_class)->finalize (object); +} + +static void +_ostree_metalink_class_init (OstreeMetalinkClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->finalize = _ostree_metalink_finalize; +} + +static void +_ostree_metalink_init (OstreeMetalink *self) +{ +} + +OstreeMetalink * +_ostree_metalink_new (OstreeFetcher *fetcher, + const char *requested_file, + guint64 max_size, + SoupURI *uri) +{ + OstreeMetalink *self = (OstreeMetalink*)g_object_new (OSTREE_TYPE_METALINK, NULL); + + self->fetcher = g_object_ref (fetcher); + self->requested_file = g_strdup (requested_file); + self->max_size = max_size; + self->uri = soup_uri_copy (uri); + + return self; +} + +static void +try_next_url (OstreeMetalinkRequest *self); + +static gboolean +valid_hex_checksum (const char *s, gsize expected_len) +{ + gsize len = strspn (s, "01234567890abcdef"); + + return len == expected_len && s[len] == '\0'; +} + +static void +on_fetched_url (GObject *src, + GAsyncResult *res, + gpointer user_data) +{ + GTask *task = user_data; + OstreeMetalinkRequest *self = g_task_get_task_data (task); + GError *local_error = NULL; + gs_unref_object GFile *result = NULL; + gs_unref_object GFileInfo *finfo = NULL; + + result = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)src, res, &local_error); + if (!result) + goto out; + + finfo = g_file_query_info (result, OSTREE_GIO_FAST_QUERYINFO, 0, + g_task_get_cancellable (task), &local_error); + if (!finfo) + goto out; + + if (g_file_info_get_size (finfo) != self->size) + { + g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Expected size is %" G_GUINT64_FORMAT " bytes but content is %" G_GUINT64_FORMAT " bytes", + self->size, g_file_info_get_size (finfo)); + goto out; + } + + if (self->verification_sha512) + { + gs_free char *actual = ot_checksum_file (result, G_CHECKSUM_SHA512, + g_task_get_cancellable (task), + &local_error); + + if (!actual) + goto out; + + if (strcmp (self->verification_sha512, actual) != 0) + { + g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Expected checksum is %s but actual is %s", + self->verification_sha512, actual); + goto out; + } + } + + if (self->verification_sha256) + { + gs_free char *actual = ot_checksum_file (result, G_CHECKSUM_SHA256, + g_task_get_cancellable (task), + &local_error); + + if (!actual) + goto out; + + if (strcmp (self->verification_sha256, actual) != 0) + { + g_set_error (&local_error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Expected checksum is %s but actual is %s", + self->verification_sha256, actual); + goto out; + } + } + + out: + if (local_error) + { + g_free (self->last_metalink_error); + self->last_metalink_error = g_strdup (local_error->message); + g_clear_error (&local_error); + + /* And here we iterate on the next one if we hit an error */ + self->current_url_index++; + try_next_url (self); + } + else + { + self->result = g_object_ref (result); + g_task_return_boolean (self->task, TRUE); + } +} + +static void +try_next_url (OstreeMetalinkRequest *self) +{ + if (self->current_url_index >= self->urls->len) + { + g_task_return_new_error (self->task, G_IO_ERROR, G_IO_ERROR_FAILED, + "Exhausted %u metalink targets, last error: %s", + self->urls->len, self->last_metalink_error); + } + else + { + SoupURI *next = self->urls->pdata[self->current_url_index]; + + _ostree_fetcher_request_uri_with_partial_async (self->metalink->fetcher, next, + self->metalink->max_size, + g_task_get_cancellable (self->task), + on_fetched_url, self->task); + } +} + +static gboolean +start_target_request_phase (OstreeMetalinkRequest *self, + GError **error) +{ + gboolean ret = FALSE; + + if (!self->found_a_file_element) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No element found"); + goto out; + } + + if (!self->found_our_file_element) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No found", self->metalink->requested_file); + goto out; + } + + if (!(self->verification_sha256 || self->verification_sha512)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No hash for sha256 or sha512 found"); + goto out; + } + + if (self->verification_sha256 && !valid_hex_checksum (self->verification_sha256, 64)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid hash digest for sha256"); + goto out; + } + + if (self->verification_sha512 && !valid_hex_checksum (self->verification_sha512, 128)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid hash digest for sha512"); + goto out; + } + + if (self->urls->len == 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No elements found"); + goto out; + } + + try_next_url (self); + + ret = TRUE; + out: + return ret; +} + +static void +on_metalink_bytes_read (GObject *src, + GAsyncResult *result, + gpointer user_data) +{ + GError *local_error = NULL; + GTask *task = user_data; + OstreeMetalinkRequest *self = g_task_get_task_data (task); + gs_unref_bytes GBytes *bytes = NULL; + gsize len; + const guint8 *data; + + bytes = g_input_stream_read_bytes_finish ((GInputStream*)src, + result, &local_error); + if (!bytes) + goto out; + + data = g_bytes_get_data (bytes, &len); + + if (len == 0) + { + if (!start_target_request_phase (self, &local_error)) + goto out; + } + else + { + if (!g_markup_parse_context_parse (self->parser, (const char*)data, len, &local_error)) + goto out; + + g_input_stream_read_bytes_async ((GInputStream*)src, 8192, G_PRIORITY_DEFAULT, + g_task_get_cancellable (task), + on_metalink_bytes_read, task); + } + + out: + if (local_error) + g_task_return_error (task, local_error); +} + +static void +on_retrieved_metalink (GObject *src, + GAsyncResult *result, + gpointer user_data) +{ + GError *local_error = NULL; + GTask *task = user_data; + gs_unref_object GInputStream *metalink_stream = NULL; + + metalink_stream = _ostree_fetcher_stream_uri_finish ((OstreeFetcher*)src, result, &local_error); + if (!metalink_stream) + goto out; + + g_input_stream_read_bytes_async (metalink_stream, 8192, G_PRIORITY_DEFAULT, + g_task_get_cancellable (task), + on_metalink_bytes_read, task); + + out: + if (local_error) + g_task_return_error (task, local_error); +} + +static void +ostree_metalink_request_unref (gpointer data) +{ + OstreeMetalinkRequest *request = data; + g_object_unref (request->metalink); + g_clear_object (&request->result); + g_free (request->last_metalink_error); + g_ptr_array_unref (request->urls); + g_free (request); +} + +static const GMarkupParser metalink_parser = { + metalink_parser_start, + metalink_parser_end, + metalink_parser_text, + NULL, + NULL +}; + +void +_ostree_metalink_request_async (OstreeMetalink *self, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data) +{ + GTask *task = g_task_new (self, cancellable, callback, user_data); + OstreeMetalinkRequest *request = g_new0 (OstreeMetalinkRequest, 1); + + request->metalink = g_object_ref (self); + request->urls = g_ptr_array_new_with_free_func ((GDestroyNotify) soup_uri_free); + request->task = task; /* Unowned */ + + request->parser = g_markup_parse_context_new (&metalink_parser, G_MARKUP_PREFIX_ERROR_POSITION, task, NULL); + + g_task_set_task_data (task, request, ostree_metalink_request_unref); + _ostree_fetcher_stream_uri_async (self->fetcher, self->uri, + self->max_size, cancellable, + on_retrieved_metalink, task); +} + +gboolean +_ostree_metalink_request_finish (OstreeMetalink *self, + GAsyncResult *result, + SoupURI **out_target_uri, + GFile **out_data, + GError **error) +{ + OstreeMetalinkRequest *request; + + g_return_val_if_fail (g_task_is_valid (result, self), FALSE); + + request = g_task_get_task_data ((GTask*)result); + + if (g_task_propagate_boolean ((GTask*)result, error)) + { + g_assert_cmpint (request->current_url_index, <, request->urls->len); + *out_target_uri = request->urls->pdata[request->current_url_index]; + *out_data = g_object_ref (request->result); + return TRUE; + } + else + return FALSE; +} + +SoupURI * +_ostree_metalink_get_uri (OstreeMetalink *self) +{ + return self->uri; +} diff --git a/src/libostree/ostree-metalink.h b/src/libostree/ostree-metalink.h new file mode 100644 index 00000000..0c26adeb --- /dev/null +++ b/src/libostree/ostree-metalink.h @@ -0,0 +1,66 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2014 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#pragma once + +#ifndef __GI_SCANNER__ + +#include "ostree-fetcher.h" + +G_BEGIN_DECLS + +#define OSTREE_TYPE_METALINK (_ostree_metalink_get_type ()) +#define OSTREE_METALINK(o) (G_TYPE_CHECK_INSTANCE_CAST ((o), OSTREE_TYPE_METALINK, OstreeMetalink)) +#define OSTREE_METALINK_CLASS(k) (G_TYPE_CHECK_CLASS_CAST((k), OSTREE_TYPE_METALINK, OstreeMetalinkClass)) +#define OSTREE_IS_METALINK(o) (G_TYPE_CHECK_INSTANCE_TYPE ((o), OSTREE_TYPE_METALINK)) +#define OSTREE_IS_METALINK_CLASS(k) (G_TYPE_CHECK_CLASS_TYPE ((k), OSTREE_TYPE_METALINK)) +#define OSTREE_METALINK_GET_CLASS(o) (G_TYPE_INSTANCE_GET_CLASS ((o), OSTREE_TYPE_METALINK, OstreeMetalinkClass)) + +typedef struct OstreeMetalinkClass OstreeMetalinkClass; +typedef struct OstreeMetalink OstreeMetalink; + +struct OstreeMetalinkClass +{ + GObjectClass parent_class; +}; + +GType _ostree_metalink_get_type (void) G_GNUC_CONST; + +OstreeMetalink *_ostree_metalink_new (OstreeFetcher *fetcher, + const char *requested_file, + guint64 max_size, + SoupURI *uri); + +SoupURI *_ostree_metalink_get_uri (OstreeMetalink *self); + +void _ostree_metalink_request_async (OstreeMetalink *self, + GCancellable *cancellable, + GAsyncReadyCallback callback, + gpointer user_data); + +gboolean _ostree_metalink_request_finish (OstreeMetalink *self, + GAsyncResult *result, + SoupURI **out_target_uri, + GFile **out_data, + GError **error); + +G_END_DECLS + +#endif diff --git a/src/libostree/ostree-repo-pull.c b/src/libostree/ostree-repo-pull.c index 0ad91575..273c963c 100644 --- a/src/libostree/ostree-repo-pull.c +++ b/src/libostree/ostree-repo-pull.c @@ -26,7 +26,7 @@ #include "ostree-core-private.h" #include "ostree-repo-private.h" #include "ostree-repo-static-delta-private.h" -#include "ostree-fetcher.h" +#include "ostree-metalink.h" #include "otutil.h" typedef struct { @@ -52,7 +52,9 @@ typedef struct { gboolean gpg_verify; + GVariant *summary; GPtrArray *static_delta_metas; + GHashTable *expected_commit_sizes; /* Maps commit checksum to known size */ GHashTable *scanned_metadata; /* Maps object name to itself */ GHashTable *requested_metadata; /* Maps object name to itself */ GHashTable *requested_content; /* Maps object name to itself */ @@ -368,6 +370,49 @@ fetch_uri_contents_utf8_sync (OtPullData *pull_data, return ret; } +typedef struct +{ + OtPullData *pull_data; + SoupURI **out_target_uri; + GFile **out_data; + gboolean success; +} FetchMetalinkSyncData; + +static void +on_metalink_fetched (GObject *src, + GAsyncResult *result, + gpointer user_data) +{ + FetchMetalinkSyncData *data = user_data; + + data->success = _ostree_metalink_request_finish ((OstreeMetalink*)src, result, + data->out_target_uri, data->out_data, + data->pull_data->async_error); + g_main_loop_quit (data->pull_data->loop); +} + +static gboolean +request_metalink_sync (OtPullData *pull_data, + OstreeMetalink *metalink, + SoupURI **out_target_uri, + GFile **out_data, + GCancellable *cancellable, + GError **error) +{ + FetchMetalinkSyncData data = { 0, }; + + data.pull_data = pull_data; + data.out_target_uri = out_target_uri; + data.out_data = out_data; + + pull_data->fetching_sync_uri = _ostree_metalink_get_uri (metalink); + _ostree_metalink_request_async (metalink, cancellable, on_metalink_fetched, &data); + + run_mainloop_monitor_fetcher (pull_data); + + return data.success; +} + static void enqueue_one_object_request (OtPullData *pull_data, const char *checksum, @@ -519,6 +564,45 @@ fetch_ref_contents (OtPullData *pull_data, return ret; } +static gboolean +lookup_commit_checksum_from_summary (OtPullData *pull_data, + const char *ref, + char **out_checksum, + gsize *out_size, + GError **error) +{ + gboolean ret = FALSE; + gs_unref_variant GVariant *refs = g_variant_get_child_value (pull_data->summary, 0); + gs_unref_variant GVariant *refdata = NULL; + gs_unref_variant GVariant *reftargetdata = NULL; + gs_unref_variant GVariant *commit_data = NULL; + guint64 commit_size; + gs_unref_variant GVariant *commit_csum_v = NULL; + gs_unref_bytes GBytes *commit_bytes = NULL; + int i; + + if (!ot_variant_bsearch_str (refs, ref, &i)) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No such branch '%s' in repository summary", + ref); + goto out; + } + + refdata = g_variant_get_child_value (refs, i); + reftargetdata = g_variant_get_child_value (refdata, 1); + g_variant_get (reftargetdata, "(t@ay@a{sv})", &commit_size, &commit_csum_v, NULL); + + if (!ostree_validate_structureof_csum_v (commit_csum_v, error)) + goto out; + + ret = TRUE; + *out_checksum = ostree_checksum_from_bytes_v (commit_csum_v); + *out_size = commit_size; + out: + return ret; +} + static void content_fetch_on_write_complete (GObject *object, GAsyncResult *result, @@ -690,7 +774,8 @@ meta_fetch_on_complete (GObject *object, GError **error = &local_error; ostree_object_name_deserialize (fetch_data->object, &checksum, &objtype); - g_debug ("fetch of %s complete", ostree_object_to_string (checksum, objtype)); + g_debug ("fetch of %s%s complete", ostree_object_to_string (checksum, objtype), + fetch_data->is_detached_meta ? " (detached)" : ""); temp_path = _ostree_fetcher_request_uri_with_partial_finish ((OstreeFetcher*)object, result, error); if (!temp_path) @@ -926,9 +1011,12 @@ enqueue_one_object_request (OtPullData *pull_data, gboolean is_meta; FetchObjectData *fetch_data; gs_free char *objpath = NULL; + guint64 *expected_max_size_p; + guint64 expected_max_size; - g_debug ("queuing fetch of %s.%s", checksum, - ostree_object_type_to_string (objtype)); + g_debug ("queuing fetch of %s.%s%s", checksum, + ostree_object_type_to_string (objtype), + is_detached_meta ? " (detached)" : ""); if (is_detached_meta) { @@ -958,10 +1046,19 @@ enqueue_one_object_request (OtPullData *pull_data, fetch_data->pull_data = pull_data; fetch_data->object = ostree_object_name_serialize (checksum, objtype); fetch_data->is_detached_meta = is_detached_meta; + + expected_max_size_p = g_hash_table_lookup (pull_data->expected_commit_sizes, checksum); + if (expected_max_size_p) + expected_max_size = *expected_max_size_p; + else if (is_meta) + expected_max_size = OSTREE_MAX_METADATA_SIZE; + else + expected_max_size = 0; + _ostree_fetcher_request_uri_with_partial_async (pull_data->fetcher, obj_uri, - is_meta ? OSTREE_MAX_METADATA_SIZE : 0, - pull_data->cancellable, - is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data); + expected_max_size, + pull_data->cancellable, + is_meta ? meta_fetch_on_complete : content_fetch_on_complete, fetch_data); soup_uri_free (obj_uri); } @@ -1124,9 +1221,11 @@ ostree_repo_pull_one_dir (OstreeRepo *self, gs_free char *remote_key = NULL; gs_free char *path = NULL; gs_free char *baseurl = NULL; + gs_free char *metalink_url_str = NULL; gs_unref_hashtable GHashTable *requested_refs_to_fetch = NULL; gs_unref_hashtable GHashTable *commits_to_fetch = NULL; gs_free char *remote_mode_str = NULL; + gs_unref_object OstreeMetalink *metalink = NULL; OtPullData pull_data_real = { 0, }; OtPullData *pull_data = &pull_data_real; GKeyFile *config = NULL; @@ -1146,6 +1245,9 @@ ostree_repo_pull_one_dir (OstreeRepo *self, pull_data->repo = self; pull_data->progress = progress; + pull_data->expected_commit_sizes = g_hash_table_new_full (g_str_hash, g_str_equal, + (GDestroyNotify)g_free, + (GDestroyNotify)g_free); pull_data->scanned_metadata = g_hash_table_new_full (ostree_hash_object_name, g_variant_equal, (GDestroyNotify)g_variant_unref, NULL); pull_data->requested_content = g_hash_table_new_full (g_str_hash, g_str_equal, @@ -1167,10 +1269,6 @@ ostree_repo_pull_one_dir (OstreeRepo *self, remote_key); goto out; } - if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error)) - goto out; - - pull_data->base_uri = soup_uri_new (baseurl); #ifdef HAVE_GPGME if (!ot_keyfile_get_boolean_with_default (config, remote_key, "gpg-verify", @@ -1256,11 +1354,54 @@ ostree_repo_pull_one_dir (OstreeRepo *self, _ostree_fetcher_set_proxy (pull_data->fetcher, http_proxy); } - if (!pull_data->base_uri) + if (!ot_keyfile_get_value_with_default (config, remote_key, "metalink", + NULL, &metalink_url_str, error)) + goto out; + + if (!metalink_url_str) { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "Failed to parse url '%s'", baseurl); - goto out; + if (!repo_get_string_key_inherit (self, remote_key, "url", &baseurl, error)) + goto out; + + pull_data->base_uri = soup_uri_new (baseurl); + + if (!pull_data->base_uri) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Failed to parse url '%s'", baseurl); + goto out; + } + } + else + { + gs_unref_object GFile *metalink_data = NULL; + SoupURI *metalink_uri = soup_uri_new (metalink_url_str); + SoupURI *target_uri = NULL; + + if (!metalink_uri) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "Invalid metalink URL: %s", metalink_url_str); + goto out; + } + + metalink = _ostree_metalink_new (pull_data->fetcher, "summary", + OSTREE_MAX_METADATA_SIZE, metalink_uri); + soup_uri_free (metalink_uri); + + if (!request_metalink_sync (pull_data, metalink, &target_uri, &metalink_data, + cancellable, error)) + goto out; + + { + gs_free char *repo_base = g_path_get_dirname (soup_uri_get_path (target_uri)); + pull_data->base_uri = soup_uri_copy (target_uri); + soup_uri_set_path (pull_data->base_uri, repo_base); + } + + if (!ot_util_variant_map (metalink_data, OSTREE_SUMMARY_GVARIANT_FORMAT, FALSE, + &pull_data->summary, error)) + goto out; } if (!load_remote_repo_config (pull_data, &remote_config, cancellable, error)) @@ -1292,7 +1433,6 @@ ostree_repo_pull_one_dir (OstreeRepo *self, for (strviter = refs_to_fetch; *strviter; strviter++) { const char *branch = *strviter; - char *contents; if (ostree_validate_checksum_string (branch, NULL)) { @@ -1301,11 +1441,7 @@ ostree_repo_pull_one_dir (OstreeRepo *self, } else { - if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) - goto out; - - /* Transfer ownership of contents */ - g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); + g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL); } } } @@ -1325,14 +1461,37 @@ ostree_repo_pull_one_dir (OstreeRepo *self, for (;branches_iter && *branches_iter; branches_iter++) { const char *branch = *branches_iter; - char *contents; - if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) + g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), NULL); + } + } + + g_hash_table_iter_init (&hash_iter, requested_refs_to_fetch); + while (g_hash_table_iter_next (&hash_iter, &key, &value)) + { + const char *branch = key; + char *contents; + + if (pull_data->summary) + { + guint64 commit_size; + guint64 *malloced_size; + + if (!lookup_commit_checksum_from_summary (pull_data, branch, &contents, &commit_size, error)) goto out; - /* Transfer ownership of contents */ - g_hash_table_insert (requested_refs_to_fetch, g_strdup (branch), contents); + malloced_size = g_new0 (guint64, 1); + *malloced_size = commit_size; + g_hash_table_insert (pull_data->expected_commit_sizes, contents, malloced_size); } + else + { + if (!fetch_ref_contents (pull_data, branch, &contents, cancellable, error)) + goto out; + } + + /* Transfer ownership of contents */ + g_hash_table_replace (requested_refs_to_fetch, g_strdup (branch), contents); } /* Create the state directory here - it's new with the commitpartial code, @@ -1460,7 +1619,9 @@ ostree_repo_pull_one_dir (OstreeRepo *self, g_free (pull_data->remote_name); if (pull_data->base_uri) soup_uri_free (pull_data->base_uri); + g_clear_pointer (&pull_data->summary, (GDestroyNotify) g_variant_unref); g_clear_pointer (&pull_data->static_delta_metas, (GDestroyNotify) g_ptr_array_unref); + g_clear_pointer (&pull_data->expected_commit_sizes, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->scanned_metadata, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_content, (GDestroyNotify) g_hash_table_unref); g_clear_pointer (&pull_data->requested_metadata, (GDestroyNotify) g_hash_table_unref); diff --git a/src/libostree/ostree-repo.c b/src/libostree/ostree-repo.c index 8a85ef25..daa532e7 100644 --- a/src/libostree/ostree-repo.c +++ b/src/libostree/ostree-repo.c @@ -381,7 +381,7 @@ GS_DEFINE_CLEANUP_FUNCTION0(GKeyFile*, local_keyfile_unref, g_key_file_unref) * ostree_repo_remote_add: * @self: Repo * @name: Name of remote - * @url: URL for remote + * @url: URL for remote (if URL begins with metalink=, it will be used as such) * @options: (allow-none): GVariant of type a{sv} * @cancellable: Cancellable * @error: Error @@ -446,7 +446,11 @@ ostree_repo_remote_add (OstreeRepo *self, target_keyfile = ostree_repo_copy_config (self); } - g_key_file_set_string (target_keyfile, section, "url", url); + if (g_str_has_prefix (url, "metalink=")) + g_key_file_set_string (target_keyfile, section, "metalink", url + strlen ("metalink=")); + else + g_key_file_set_string (target_keyfile, section, "url", url); + if (options) keyfile_set_from_vardict (target_keyfile, section, options); @@ -2264,3 +2268,81 @@ out: (void) gs_file_unlink (commit_tmp_path, NULL, NULL); return ret; } + +/** + * ostree_repo_regenerate_summary: + * @self: Repo + * @additional_metadata: (allow-none): A GVariant of type a{sv}, or %NULL + * @cancellable: Cancellable + * @error: Error + * + * An OSTree repository can contain a high level "summary" file that + * describes the available branches and other metadata. + * + * It is not regenerated automatically when commits are created; this + * API is available to atomically regenerate the summary after + * multiple commits. It should only be invoked by one process at a + * time. + */ +gboolean +ostree_repo_regenerate_summary (OstreeRepo *self, + GVariant *additional_metadata, + GCancellable *cancellable, + GError **error) +{ + gboolean ret = FALSE; + gs_unref_object GFile *summary_path = NULL; + gs_unref_hashtable GHashTable *refs = NULL; + gs_unref_variant_builder GVariantBuilder *refs_builder = NULL; + gs_unref_variant GVariant *summary = NULL; + GList *ordered_keys = NULL; + GList *iter = NULL; + + if (!ostree_repo_list_refs (self, NULL, &refs, cancellable, error)) + goto out; + + refs_builder = g_variant_builder_new (G_VARIANT_TYPE ("a(s(taya{sv}))")); + + ordered_keys = g_hash_table_get_keys (refs); + ordered_keys = g_list_sort (ordered_keys, (GCompareFunc)strcmp); + + for (iter = ordered_keys; iter; iter = iter->next) + { + const char *ref = iter->data; + const char *commit = g_hash_table_lookup (refs, ref); + gs_unref_variant GVariant *commit_obj = NULL; + + g_assert (commit); + + if (!ostree_repo_load_variant (self, OSTREE_OBJECT_TYPE_COMMIT, commit, &commit_obj, error)) + goto out; + + g_variant_builder_add_value (refs_builder, + g_variant_new ("(s(t@ay@a{sv}))", ref, + g_variant_get_size (commit_obj), + ostree_checksum_to_bytes_v (commit), + ot_gvariant_new_empty_string_dict ())); + } + + { + gs_unref_variant_builder GVariantBuilder *summary_builder = + g_variant_builder_new (OSTREE_SUMMARY_GVARIANT_FORMAT); + + g_variant_builder_add_value (summary_builder, g_variant_builder_end (refs_builder)); + g_variant_builder_add_value (summary_builder, additional_metadata ? additional_metadata : ot_gvariant_new_empty_string_dict ()); + summary = g_variant_builder_end (summary_builder); + g_variant_ref_sink (summary); + } + + summary_path = g_file_get_child (self->repodir, "summary"); + + if (!ot_util_variant_save (summary_path, summary, cancellable, error)) + goto out; + + ret = TRUE; + out: + if (ordered_keys) + g_list_free (ordered_keys); + return ret; +} + diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index cace5e5b..de38ce43 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -568,5 +568,11 @@ gboolean ostree_repo_verify_commit (OstreeRepo *self, GCancellable *cancellable, GError **error); +gboolean ostree_repo_regenerate_summary (OstreeRepo *self, + GVariant *additional_metadata, + GCancellable *cancellable, + GError **error); + + G_END_DECLS diff --git a/src/libotutil/ot-checksum-utils.c b/src/libotutil/ot-checksum-utils.c index 6a45c01b..b7289929 100644 --- a/src/libotutil/ot-checksum-utils.c +++ b/src/libotutil/ot-checksum-utils.c @@ -139,6 +139,32 @@ ot_gio_checksum_stream (GInputStream *in, return ot_gio_splice_get_checksum (NULL, in, out_csum, cancellable, error); } +char * +ot_checksum_file (GFile *file, + GChecksumType checksum_type, + GCancellable *cancellable, + GError **error) +{ + GChecksum *checksum = NULL; + char *ret = NULL; + gs_unref_object GInputStream *in = NULL; + + in = (GInputStream*)g_file_read (file, cancellable, error); + if (!in) + goto out; + + checksum = g_checksum_new (checksum_type); + + if (!ot_gio_splice_update_checksum (NULL, in, checksum, cancellable, error)) + goto out; + + ret = g_strdup (g_checksum_get_string (checksum)); + out: + g_clear_pointer (&checksum, (GDestroyNotify) g_checksum_free); + return ret; + +} + static void checksum_stream_thread (GSimpleAsyncResult *result, GObject *object, diff --git a/src/libotutil/ot-checksum-utils.h b/src/libotutil/ot-checksum-utils.h index 7778ed09..eb8bbc04 100644 --- a/src/libotutil/ot-checksum-utils.h +++ b/src/libotutil/ot-checksum-utils.h @@ -53,6 +53,11 @@ gboolean ot_gio_checksum_stream (GInputStream *in, GCancellable *cancellable, GError **error); +char * ot_checksum_file (GFile *file, + GChecksumType checksum_type, + GCancellable *cancellable, + GError **error); + void ot_gio_checksum_stream_async (GInputStream *in, int io_priority, GCancellable *cancellable, diff --git a/src/libotutil/ot-variant-utils.c b/src/libotutil/ot-variant-utils.c index ffc2defa..aa3dff56 100644 --- a/src/libotutil/ot-variant-utils.c +++ b/src/libotutil/ot-variant-utils.c @@ -30,6 +30,12 @@ #include "otutil.h" +GVariant * +ot_gvariant_new_empty_string_dict (void) +{ + return g_variant_builder_end (g_variant_builder_new (G_VARIANT_TYPE ("a{sv}"))); +} + GVariant * ot_gvariant_new_bytearray (const guchar *data, gsize len) @@ -282,3 +288,61 @@ ot_variant_new_from_bytes (const GVariantType *type, (GDestroyNotify)g_bytes_unref, bytes); #endif } + +/** + * ot_variant_bsearch_str: + * @array: A GVariant array whose first element must be a string + * @str: Search for this string + * @out_pos: Output position + * + * + * Binary search in a GVariant array, which must be of the form 'a(s...)', + * where '...' may be anything. The array elements must be sorted. + * + * Returns: %TRUE if found, %FALSE otherwise + */ +gboolean +ot_variant_bsearch_str (GVariant *array, + const char *str, + int *out_pos) +{ + gsize imax, imin; + gsize imid; + gsize n; + + n = g_variant_n_children (array); + if (n == 0) + return FALSE; + + imax = n - 1; + imin = 0; + while (imax >= imin) + { + gs_unref_variant GVariant *child = NULL; + const char *cur; + int cmp; + + imid = (imin + imax) / 2; + + child = g_variant_get_child_value (array, imid); + g_variant_get_child (child, 0, "&s", &cur, NULL); + + cmp = strcmp (cur, str); + if (cmp < 0) + imin = imid + 1; + else if (cmp > 0) + { + if (imid == 0) + break; + imax = imid - 1; + } + else + { + *out_pos = imid; + return TRUE; + } + } + + *out_pos = imid; + return FALSE; +} diff --git a/src/libotutil/ot-variant-utils.h b/src/libotutil/ot-variant-utils.h index 218e5439..422a8037 100644 --- a/src/libotutil/ot-variant-utils.h +++ b/src/libotutil/ot-variant-utils.h @@ -31,6 +31,8 @@ GVariant *ot_gvariant_new_bytearray (const guchar *data, GVariant *ot_gvariant_new_ay_bytes (GBytes *bytes); +GVariant *ot_gvariant_new_empty_string_dict (void); + GHashTable *ot_util_variant_asv_to_hash_table (GVariant *variant); GVariant * ot_util_variant_take_ref (GVariant *variant); @@ -70,5 +72,10 @@ ot_variant_new_from_bytes (const GVariantType *type, GBytes *bytes, gboolean trusted); +gboolean +ot_variant_bsearch_str (GVariant *array, + const char *str, + int *out_pos); + G_END_DECLS diff --git a/src/ostree/main.c b/src/ostree/main.c index b16d8c12..e114690d 100644 --- a/src/ostree/main.c +++ b/src/ostree/main.c @@ -55,6 +55,7 @@ static OstreeCommand commands[] = { { "rev-parse", ostree_builtin_rev_parse, 0 }, { "show", ostree_builtin_show, 0 }, { "static-delta", ostree_builtin_static_delta, 0 }, + { "summary", ostree_builtin_summary, 0 }, #ifdef HAVE_LIBSOUP { "trivial-httpd", ostree_builtin_trivial_httpd, OSTREE_BUILTIN_FLAG_NO_REPO }, #endif diff --git a/src/ostree/ot-builtin-summary.c b/src/ostree/ot-builtin-summary.c new file mode 100644 index 00000000..2f9cae5b --- /dev/null +++ b/src/ostree/ot-builtin-summary.c @@ -0,0 +1,63 @@ +/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*- + * + * Copyright (C) 2014 Colin Walters + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + */ + +#include "config.h" + +#include "ot-builtins.h" +#include "ostree.h" +#include "otutil.h" + +static gboolean opt_update; + +static GOptionEntry options[] = { + { "update", 'u', 0, G_OPTION_ARG_NONE, &opt_update, "Update the summary", NULL }, + { NULL } +}; + +gboolean +ostree_builtin_summary (int argc, char **argv, OstreeRepo *repo, GCancellable *cancellable, GError **error) +{ + gboolean ret = FALSE; + GOptionContext *context; + + context = g_option_context_new ("Manage summary metadata"); + g_option_context_add_main_entries (context, options, NULL); + + if (!g_option_context_parse (context, &argc, &argv, error)) + goto out; + + if (opt_update) + { + if (!ostree_repo_regenerate_summary (repo, NULL, cancellable, error)) + goto out; + } + else + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "No option specified; use -u to update summary"); + goto out; + } + + ret = TRUE; + out: + if (context) + g_option_context_free (context); + return ret; +} diff --git a/src/ostree/ot-builtins.h b/src/ostree/ot-builtins.h index fee66f2b..b8b6507f 100644 --- a/src/ostree/ot-builtins.h +++ b/src/ostree/ot-builtins.h @@ -46,6 +46,7 @@ BUILTINPROTO(reset); BUILTINPROTO(fsck); BUILTINPROTO(show); BUILTINPROTO(static_delta); +BUILTINPROTO(summary); BUILTINPROTO(rev_parse); BUILTINPROTO(remote); BUILTINPROTO(write_refs); diff --git a/tests/test-pull-metalink.sh b/tests/test-pull-metalink.sh new file mode 100755 index 00000000..3d685788 --- /dev/null +++ b/tests/test-pull-metalink.sh @@ -0,0 +1,120 @@ +#!/bin/bash +# +# Copyright (C) 2014 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -e + +. $(dirname $0)/libtest.sh + +setup_fake_remote_repo1 "archive-z2" + +# And another web server acting as the metalink server +cd ${test_tmpdir} +mkdir metalink-data +cd metalink-data +ostree trivial-httpd --daemonize -p ${test_tmpdir}/metalink-httpd-port +metalink_port=$(cat ${test_tmpdir}/metalink-httpd-port) +echo "http://127.0.0.1:${metalink_port}" > ${test_tmpdir}/metalink-httpd-address + +ostree --repo=${test_tmpdir}/ostree-srv/gnomerepo summary -u + +summary_path=${test_tmpdir}/ostree-srv/gnomerepo/summary + +echo -n broken > ${summary_path}.bad + +echo '1..1' +cd ${test_tmpdir} + +cat > ${test_tmpdir}/metalink-data/metalink.xml < + + + + $(stat -c '%s' ${summary_path}) + + $(md5sum ${summary_path} | cut -f 1 -d ' ') + $(sha256sum ${summary_path} | cut -f 1 -d ' ') + $(sha512sum ${summary_path} | cut -f 1 -d ' ') + + + $(cat httpd-address)/ostree/gnomerepo/summary.bad + $(cat httpd-address)/ostree/gnomerepo/nosuchfile + $(cat httpd-address)/ostree/gnomerepo/summary + + + + +EOF + +cd ${test_tmpdir} +mkdir repo +${CMD_PREFIX} ostree --repo=repo init +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat metalink-httpd-address)/metalink.xml +${CMD_PREFIX} ostree --repo=repo pull origin:main +${CMD_PREFIX} ostree --repo=repo rev-parse origin:main +${CMD_PREFIX} ostree --repo=repo fsck +echo "ok pull via metalink" + +cp metalink-data/metalink.xml{,.orig} +cp ostree-srv/gnomerepo/summary{,.orig} + +test_metalink_pull_error() { + msg=$1 + rm repo -rf + mkdir repo + ${CMD_PREFIX} ostree --repo=repo init + ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin metalink=$(cat metalink-httpd-address)/metalink.xml + if ${CMD_PREFIX} ostree --repo=repo pull origin:main 2>err.txt; then + assert_not_reached "pull unexpectedly succeeded" + fi + cat err.txt + assert_file_has_content err.txt "${msg}" +} + +cd ${test_tmpdir} +sed -e 's,.*,bacon,' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml +test_metalink_pull_error "Invalid hash digest for sha512" +echo "ok metalink err hash format" + +cd ${test_tmpdir} +sed -e 's,.*,'$( (echo -n dummy; cat ${summary_path}) | sha512sum | cut -f 1 -d ' ')',' < metalink-data/metalink.xml.orig > metalink-data/metalink.xml +test_metalink_pull_error "Expected checksum is .* but actual is" +echo "ok metalink err hash sha512" + +cd ${test_tmpdir} +cp metalink-data/metalink.xml.orig metalink-data/metalink.xml +echo -n moo > ostree-srv/gnomerepo/summary +test_metalink_pull_error "Expected size is .* bytes but content is 3 bytes" +echo "ok metalink err size" +cp ostree-srv/gnomerepo/summary{.orig,} + +cd ${test_tmpdir} +grep -v sha256 < metalink-data/metalink.xml.orig |grep -v sha512 > metalink-data/metalink.xml +test_metalink_pull_error "No.*verification.*with known.*hash" +echo "ok metalink err no verify" + +cd ${test_tmpdir} +grep -v ' metalink-data/metalink.xml +test_metalink_pull_error "No.*url.*method.*elements" +echo "ok metalink err no url" + +cd ${test_tmpdir} +echo bacon > metalink-data/metalink.xml +test_metalink_pull_error "Document must begin with an element" +echo "ok metalink err malformed" +