Add support for pulling from remote archives

This necessitated a large set of changes.

We now support an "archive" mode for repositories.  In this mode,
files are stored "packed" rather than hard linked.  This allows one to
e.g. store an OSTree repository with root-owned files as non-root.  It
is also used as the basis for serving repositories via HTTP.

While doing this I realized that GVariant is endianness-dependent; I
decided to just store all data in big endian.

tests/.gitignore

@@ -1 +1,4 @@
 !Makefile
+ostree-http-server
+run-apache
+tmpdir-lifecycle

tests/Makefile

@@ -20,7 +20,13 @@
 
 TESTS = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
 
-all:
+all: tmpdir-lifecycle run-apache
+
+tmpdir-lifecycle: tmpdir-lifecycle.c Makefile
+	gcc $(CFLAGS) `pkg-config --cflags --libs gio-unix-2.0` -o $@ $<
+
+run-apache: run-apache.c Makefile
+	gcc $(CFLAGS) `pkg-config --cflags --libs gio-unix-2.0` -o $@ $<
 
 check:
 	@for test in $(TESTS); do \

tests/libtest.sh

@@ -18,6 +18,9 @@
 #
 # Author: Colin Walters <walters@verbum.org>
 
+cd `dirname $0`
+SRCDIR=`pwd`
+cd -
 TMPDIR=${TMPDIR:-/tmp}
 export TMPDIR
 test_tmpdir=`mktemp -d "$TMPDIR/ostree-tests.XXXXXXXXXX"`
@@ -91,4 +94,53 @@ setup_test_repository2 () {
     ostree fsck -q $ot_repo
 }
 
+setup_fake_remote_repo1() {
+    oldpwd=`pwd`
+    mkdir ostree-srv
+    cd ostree-srv
+    mkdir gnomerepo
+    ostree init --archive --repo=gnomerepo
+    mkdir gnomerepo-files
+    cd gnomerepo-files 
+    echo first > firstfile
+    mkdir baz
+    echo moo > baz/cow
+    echo alien > baz/saucer
+    find | grep -v '^\.$' | ostree commit --repo=${test_tmpdir}/ostree-srv/gnomerepo -b main -s "A remote commit" -m "Some Commit body" --from-stdin
+    mkdir baz/deeper
+    ostree commit --repo=${test_tmpdir}/ostree-srv/gnomerepo -b main -s "Add deeper" --add=baz/deeper
+    echo hi > baz/deeper/ohyeah
+    mkdir baz/another/
+    echo x > baz/another/y
+    find | grep -v '^\.$' | ostree commit --repo=${test_tmpdir}/ostree-srv/gnomerepo -b main -s "The rest" --from-stdin
+    cd ..
+    rm -rf gnomerepo-files
+    
+    cd ${test_tmpdir}
+    mkdir ${test_tmpdir}/httpd
+    cd httpd
+    cp $(command -v ostree-http-backend) .
+    chmod a+x ostree-http-backend
+    cat >httpd.conf <<EOF
+ServerRoot ${test_tmpdir}/httpd
+PidFile pid
+LogLevel crit
+ErrorLog log
+LockFile lock
+ServerName localhost
+
+LoadModule alias_module modules/mod_alias.so
+LoadModule cgi_module modules/mod_cgi.so
+LoadModule env_module modules/mod_env.so
+
+StartServers 1
+
+# SetEnv OSTREE_REPO_PREFIX ${test_tmpdir}/ostree-srv
+Alias /ostree/ ${test_tmpdir}/ostree-srv/
+# ScriptAlias /ostree/  ${test_tmpdir}/httpd/ostree-http-backend/
+EOF
+    ${SRCDIR}/tmpdir-lifecycle ${SRCDIR}/run-apache `pwd`/httpd.conf ${test_tmpdir}/httpd-address
+    cd ${oldpwd} 
+}
+
 trap 'die' EXIT

tests/run-apache.c

@@ -0,0 +1,167 @@
+/* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Copyright (C) 2011 Colin Walters <walters@verbum.org>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Author: Colin Walters <walters@verbum.org>
+ */
+
+#include <gio/gio.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <sys/stat.h>
+
+/* Taken from gnome-user-share src/httpd.c under the GPLv2 */
+static int
+get_port (void)
+{
+  int sock;
+  int saved_errno;
+  struct sockaddr_in addr;
+  int reuse;
+  socklen_t len;
+
+  sock = socket (PF_INET, SOCK_STREAM, 0);
+  if (sock < 0)
+    {
+      return -1;
+    }
+  
+  memset (&addr, 0, sizeof (addr));
+  addr.sin_family = AF_INET;
+  addr.sin_port = 0;
+  addr.sin_addr.s_addr = htonl (INADDR_LOOPBACK);
+
+  reuse = 1;
+  setsockopt (sock, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof (reuse));
+  if (bind (sock, (struct sockaddr *)&addr, sizeof (addr)) == -1)
+    {
+      saved_errno = errno;
+      close (sock);
+      errno = saved_errno;
+      return -1;
+    }
+
+  len = sizeof (addr);
+  if (getsockname (sock, (struct sockaddr *)&addr, &len) == -1)
+    {
+      saved_errno = errno;
+      close (sock);
+      errno = saved_errno;
+      return -1;
+    }
+
+#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__APPLE__)
+  /* XXX This exposes a potential race condition, but without this,
+   * httpd will not start on the above listed platforms due to the fact
+   * that SO_REUSEADDR is also needed when Apache binds to the listening
+   * socket.  At this time, Apache does not support that socket option.
+   */
+  close (sock);
+#endif
+  return ntohs (addr.sin_port);
+}
+
+static const char *known_httpd_modules_locations [] = {
+  "/usr/libexec/apache2",
+  "/usr/lib/apache2/modules",
+  "/usr/lib64/httpd/modules",
+  "/usr/lib/httpd/modules",
+  NULL
+};
+
+static gchar*
+get_httpd_modules_path ()
+{
+  int i;
+
+  for (i = 0; known_httpd_modules_locations[i]; i++)
+    {
+      if (g_file_test (known_httpd_modules_locations[i], G_FILE_TEST_IS_EXECUTABLE)
+	  && g_file_test (known_httpd_modules_locations[i], G_FILE_TEST_IS_DIR))
+	{
+	  return g_strdup (known_httpd_modules_locations[i]);
+	}
+    }
+  return NULL;
+}
+
+int
+main (int     argc,
+      char  **argv)
+{
+  int port;
+  char *listen;
+  char *address_string;
+  GError *error = NULL;
+  GPtrArray *httpd_argv;
+  char *modules;
+
+  if (argc != 3)
+    {
+      fprintf (stderr, "usage: run-apache CONF PORTFILE");
+      return 1;
+    }
+
+  g_type_init ();
+
+  port = get_port ();
+  if (port == -1)
+    {
+      perror ("Failed to bind port");
+      return 1;
+    }
+
+  httpd_argv = g_ptr_array_new ();
+  g_ptr_array_add (httpd_argv, "httpd");
+  g_ptr_array_add (httpd_argv, "-f");
+  g_ptr_array_add (httpd_argv, argv[1]);
+  g_ptr_array_add (httpd_argv, "-C");
+  listen = g_strdup_printf ("Listen 127.0.0.1:%d", port);
+  g_ptr_array_add (httpd_argv, listen);
+  g_ptr_array_add (httpd_argv, NULL);
+
+  address_string = g_strdup_printf ("http://127.0.0.1:%d\n", port);
+  
+  if (!g_file_set_contents (argv[2], address_string, -1, &error))
+    {
+      g_printerr ("%s\n", error->message);
+      return 1;
+    }
+
+  setenv ("LANG", "C", 1);
+  modules = get_httpd_modules_path ();
+  if (modules == NULL)
+    {
+      g_printerr ("Failed to find httpd modules\n");
+      return 1;
+    }
+  if (symlink (modules, "modules") < 0)
+    {
+      perror ("failed to make modules symlink");
+      return 1;
+    }
+  execvp ("httpd", (char**)httpd_argv->pdata);
+  perror ("Failed to run httpd");
+  return 1;
+}

tests/t0012-pull.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# Author: Colin Walters <walters@verbum.org>
+
+set -e
+
+. libtest.sh
+
+echo '1..1'
+
+setup_fake_remote_repo1
+cd ${test_tmpdir}
+mkdir repo
+ostree init --repo=repo
+ostree remote --repo=repo add origin $(cat httpd-address)/ostree/gnomerepo
+ostree pull --repo=repo origin main
+echo "ok pull"

tests/t0013-commit-archive.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# Copyright (C) 2011 Colin Walters <walters@verbum.org>
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# Author: Colin Walters <walters@verbum.org>
+
+set -e
+
+. libtest.sh
+
+echo '1..4'
+
+mkdir files
+cd files
+echo first > firstfile
+echo second > secondfile
+ln -s foo bar
+
+mkdir ../repo
+repo="--repo=../repo"
+ostree init --archive $repo
+echo 'ok init'
+ostree commit $repo -b test -s "Test Commit 1" -m "Commit body first" --add=firstfile
+echo 'ok commit 1'
+ostree commit $repo -b test -s "Test Commit 2" -m "Commit body first" --add=secondfile --add=bar
+echo 'ok commit 2'
+ostree fsck -q $repo
+echo 'ok fsck'

tests/tmpdir-lifecycle.c

@@ -1,4 +1,6 @@
 /* -*- mode: C; c-file-style: "gnu"; indent-tabs-mode: nil; -*-
+ *
+ * Kill a child process when the current directory is deleted
  *
  * Copyright (C) 2011 Colin Walters <walters@verbum.org>
  *
@@ -19,35 +21,16 @@
  * Author: Colin Walters <walters@verbum.org>
  */
 
-#include <libsoup/soup-gnome.h>
-#include <sys/types.h>
+#include <gio/gio.h>
 #include <unistd.h>
 #include <stdlib.h>
+#include <string.h>
 
-static void
-request_callback (SoupServer *server, SoupMessage *msg,
-		  const char *path, GHashTable *query,
-		  SoupClientContext *context, gpointer data)
-{
-  if (msg->method == SOUP_METHOD_GET) 
-    {
-      GFile *file;
-      char *content;
-      gsize len;
-      
-      /* Strip leading / */
-      file = g_file_new_for_path (path + 1);
-      
-      if (g_file_load_contents (file, NULL, &content, &len, NULL, NULL))
-	soup_message_set_response (msg, "application/octet-stream", SOUP_MEMORY_TAKE, content, len);
-      else
-	soup_message_set_status (msg, SOUP_STATUS_NOT_FOUND);
-    }
-  else
-    {
-      soup_message_set_status (msg, SOUP_STATUS_METHOD_NOT_ALLOWED);
-    }
-}
+struct TmpdirLifecyleData {
+  GMainLoop *loop;
+  GPid pid;
+  gboolean exited;
+};
 
 static void
 on_dir_changed (GFileMonitor  *mon,
@@ -56,49 +39,65 @@ on_dir_changed (GFileMonitor  *mon,
 		GFileMonitorEvent  event,
 		gpointer user_data)
 {
-  GMainLoop *loop = user_data;
+  struct TmpdirLifecyleData *data = user_data;
 
   if (event == G_FILE_MONITOR_EVENT_DELETED)
-    g_main_loop_quit (loop);
+    g_main_loop_quit (data->loop);
+}
+
+static void
+on_child_exited (GPid  pid,
+                 int status,
+                 gpointer user_data)
+{
+  struct TmpdirLifecyleData *data = user_data;
+
+  data->exited = TRUE;
+  g_main_loop_quit (data->loop);
 }
 
 int
 main (int     argc,
       char  **argv)
 {
-  SoupAddress *addr;
-  SoupServer *server;
-  GMainLoop *loop;
   GFileMonitor *monitor;
   GFile *curdir;
   GError *error = NULL;
+  GPtrArray *new_argv;
+  int i;
+  GPid pid;
+  struct TmpdirLifecyleData data;
 
   g_type_init ();
 
-  loop = g_main_loop_new (NULL, TRUE);
+  memset (&data, 0, sizeof (data));
+
+  data.loop = g_main_loop_new (NULL, TRUE);
 
   curdir = g_file_new_for_path (".");
   monitor = g_file_monitor_directory (curdir, 0, NULL, &error);
   if (!monitor)
     exit (1);
   g_signal_connect (monitor, "changed",
-		    G_CALLBACK (on_dir_changed), loop);
+		    G_CALLBACK (on_dir_changed), &data);
 
-  addr = soup_address_new ("127.0.0.1", SOUP_ADDRESS_ANY_PORT);
-  soup_address_resolve_sync (addr, NULL);
+  new_argv = g_ptr_array_new ();
+  for (i = 1; i < argc; i++)
+    g_ptr_array_add (new_argv, argv[i]);
+  g_ptr_array_add (new_argv, NULL);
 
-  server = soup_server_new (SOUP_SERVER_INTERFACE, addr,
-			    SOUP_SERVER_ASYNC_CONTEXT, g_main_loop_get_context (loop),
-			    NULL);
-  soup_server_add_handler (server, NULL,
-			   request_callback,
-			   NULL, NULL);
+  if (!g_spawn_async (NULL, (char**)new_argv->pdata, NULL, G_SPAWN_SEARCH_PATH | G_SPAWN_DO_NOT_REAP_CHILD,
+                      NULL, NULL, &pid, &error))
+    {
+      g_printerr ("%s\n", error->message);
+      return 1;
+    }
+  g_child_watch_add (pid, on_child_exited, &data);
 
-  soup_server_run_async (server);
+  g_main_loop_run (data.loop);
 
-  g_print ("http://127.0.0.1:%ld\n", (long)soup_server_get_port (server));
-
-  g_main_loop_run (loop);
+  if (!data.exited)
+    kill (data.pid, SIGTERM);
 
   return 0;
 }