ostree/pull: Add network-retries command line option

This exposes a way to specify from the command line the number
of times to retry each download after a network error. If a negative
value is given, then the default number of retries (5) is used. If 0
is given, then errors are returned without retrying.

closes #1659

Closes: #1669
Approved by: jlebon

tests/test-pull-repeated.sh

@@ -23,7 +23,7 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
-echo "1..2"
+echo "1..4"
 
 COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}"
 
@@ -47,7 +47,20 @@ ${CMD_PREFIX} ostree --repo=repo rev-parse main
 popd
 echo "ok repeated pull after 500s"
 
-# Now test from a repo which gives error 408 (request timeout) a lot of the time.
+# Sanity check with no network retries and 408s given, pull should fail.
+rm ostree-srv httpd repo -rf
+setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-408s=99
+
+pushd ${test_tmpdir}
+ostree_repo_init repo --mode=archive
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+assert_fail ${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=0 main 2>err.txt
+assert_file_has_content err.txt "\(408.*Request Timeout\)\|\(HTTP 408\)"
+
+popd
+echo "ok no retries after a 408"
+
+# Test pulling a repo which gives error 408 (request timeout) a lot of the time.
 rm ostree-srv httpd repo -rf
 setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-408s=50
 
@@ -55,7 +68,7 @@ pushd ${test_tmpdir}
 ostree_repo_init repo --mode=archive
 ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
 for x in $(seq 40); do
-    if ${CMD_PREFIX} ostree --repo=repo pull --mirror origin main 2>err.txt; then
+    if ${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=2 main 2>err.txt; then
        echo "Success on iteration ${x}"
        break;
     fi
@@ -67,3 +80,22 @@ ${CMD_PREFIX} ostree --repo=repo rev-parse main
 
 popd
 echo "ok repeated pull after 408s"
+
+# Test pulling a repo that gives 408s a lot of the time, with many network retries.
+rm ostree-srv httpd repo -rf
+setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-408s=50
+
+pushd ${test_tmpdir}
+ostree_repo_init repo --mode=archive
+${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
+
+# Using 8 network retries gives error rate of <0.5%, when --random-408s=50
+if ${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=8 main 2>err.txt; then
+   echo "Success with big number of network retries"
+fi
+
+${CMD_PREFIX} ostree --repo=repo fsck
+${CMD_PREFIX} ostree --repo=repo rev-parse main
+
+popd
+echo "ok big number of retries with one 408"