tests: Add some test coverage of repeated pulls w/HTTP 500s

Systems like pulp may want to keep retrying in a loop if the server
throws a (hopefully transient) 500, and we need test coverage of
handling these errors versus our existing 404 and 206 coverage.

Closes: #383
Approved by: mbarnes

Makefile-tests.am

@@ -57,6 +57,7 @@ dist_test_scripts = \
 	tests/test-pull-metalink.sh \
 	tests/test-pull-summary-sigs.sh \
 	tests/test-pull-resume.sh \
+	tests/test-pull-repeated.sh \
 	tests/test-pull-untrusted.sh \
 	tests/test-pull-override-url.sh \
 	tests/test-local-pull.sh \

src/ostree/ot-builtin-trivial-httpd.c

@@ -38,8 +38,14 @@ static char *opt_log = NULL;
 static gboolean opt_daemonize;
 static gboolean opt_autoexit;
 static gboolean opt_force_ranges;
+static int opt_random_500s_percentage;
+/* We have a strong upper bound for any unlikely
+ * cases involving repeated random 500s. */
+static int opt_random_500s_max = 100;
 static gint opt_port = 0;
 
+static guint emitted_random_500s_count = 0;
+
 typedef struct {
   GFile *root;
   gboolean running;
@@ -52,6 +58,8 @@ static GOptionEntry options[] = {
   { "port", 'P', 0, G_OPTION_ARG_INT, &opt_port, "Use the specified TCP port", NULL },
   { "port-file", 'p', 0, G_OPTION_ARG_FILENAME, &opt_port_file, "Write port number to PATH (- for standard output)", "PATH" },
   { "force-range-requests", 0, 0, G_OPTION_ARG_NONE, &opt_force_ranges, "Force range requests by only serving half of files", NULL },
+  { "random-500s", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_percentage, "Generate random HTTP 500 errors approximately for PERCENTAGE requests", "PERCENTAGE" },
+  { "random-500s-max", 0, 0, G_OPTION_ARG_INT, &opt_random_500s_max, "Limit HTTP 500 errors to MAX (default 100)", "MAX" },
   { "log-file", 0, 0, G_OPTION_ARG_FILENAME, &opt_log, "Put logs here", "PATH" },
   { NULL }
 };
@@ -187,6 +195,15 @@ do_get (OtTrivialHttpd    *self,
       goto out;
     }
 
+  if (opt_random_500s_percentage > 0 &&
+      emitted_random_500s_count < opt_random_500s_max &&
+      g_random_int_range (0, 100) < opt_random_500s_percentage)
+    {
+      emitted_random_500s_count++;
+      soup_message_set_status (msg, SOUP_STATUS_INTERNAL_SERVER_ERROR);
+      goto out;
+    }
+
   if (path[0] == '/')
     path++;
 
@@ -388,6 +405,13 @@ ostree_builtin_trivial_httpd (int argc, char **argv, GCancellable *cancellable,
 
   app->root = g_file_new_for_path (dirpath);
 
+  if (!(opt_random_500s_percentage >= 0 && opt_random_500s_percentage <= 99))
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "Invalid --random-500s=%u", opt_random_500s_percentage);
+      goto out;
+    }
+
   if (opt_log)
     {
       GOutputStream *stream = NULL;

tests/test-pull-repeated.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+#
+# Copyright (C) 2016 Red Hat
+#
+# This library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2 of the License, or (at your option) any later version.
+#
+# This library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this library; if not, write to the
+# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+# Boston, MA 02111-1307, USA.
+
+set -euo pipefail
+
+. $(dirname $0)/libtest.sh
+
+echo "1..1"
+
+COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}"
+setup_fake_remote_repo1 "archive-z2" "${COMMIT_SIGN}" --random-500s=50
+
+cd ${test_tmpdir}
+${CMD_PREFIX} ostree --repo=repo init --mode=archive-z2
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+for x in $(seq 200); do
+    if ${CMD_PREFIX} ostree --repo=repo pull --mirror origin main 2>err.txt; then
+	echo "Success on iteration ${x}"
+	break;
+    fi
+    assert_file_has_content err.txt "500.*Internal Server Error"
+done
+
+${CMD_PREFIX} ostree --repo=repo fsck
+${CMD_PREFIX} ostree --repo=repo rev-parse main
+
+echo "ok repeated pull after 500s"