From 722be7e3d74f095db749a9c93df4c618f67371bd Mon Sep 17 00:00:00 2001 From: Ricardo Noriega Date: Fri, 28 Oct 2022 13:59:08 +0200 Subject: [PATCH 1/2] Enabling retry for HTTP 500 internal server error Signed-off-by: Ricardo Noriega Use G_IO_ERROR_BUSY to represent 500 error Signed-off-by: Ricardo Noriega Make G_IO_ERROR_BUSY independent from glib version Signed-off-by: Ricardo Noriega --- src/libostree/ostree-fetcher-util.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libostree/ostree-fetcher-util.c b/src/libostree/ostree-fetcher-util.c index 450a3abe..1b317630 100644 --- a/src/libostree/ostree-fetcher-util.c +++ b/src/libostree/ostree-fetcher-util.c @@ -239,6 +239,7 @@ _ostree_fetcher_should_retry_request (const GError *error, g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_NOT_FOUND) || g_error_matches (error, G_IO_ERROR, G_IO_ERROR_HOST_UNREACHABLE) || g_error_matches (error, G_IO_ERROR, G_IO_ERROR_PARTIAL_INPUT) || + g_error_matches (error, G_IO_ERROR, G_IO_ERROR_BUSY) || #if !GLIB_CHECK_VERSION(2, 44, 0) g_error_matches (error, G_IO_ERROR, G_IO_ERROR_BROKEN_PIPE) || #else @@ -269,6 +270,8 @@ _ostree_fetcher_http_status_code_to_io_error (guint status_code) return G_IO_ERROR_NOT_FOUND; case 408: /* SOUP_STATUS_REQUEST_TIMEOUT */ return G_IO_ERROR_TIMED_OUT; + case 500: /* SOUP_STATUS_INTERNAL_SERVER_ERROR */ + return G_IO_ERROR_BUSY; default: return G_IO_ERROR_FAILED; } From 6ce2324e4a0e75d1546837b606efad18bb71a59f Mon Sep 17 00:00:00 2001 From: Ricardo Noriega Date: Wed, 9 Nov 2022 10:52:16 +0100 Subject: [PATCH 2/2] Add tests for cases under random 500s Signed-off-by: Ricardo Noriega --- tests/test-pull-repeated.sh | 44 ++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/test-pull-repeated.sh b/tests/test-pull-repeated.sh index 4c321618..7f724c95 100755 --- a/tests/test-pull-repeated.sh +++ b/tests/test-pull-repeated.sh @@ -26,18 +26,32 @@ if has_gpgme; then COMMIT_SIGN="--gpg-homedir=${TEST_GPG_KEYHOME} --gpg-sign=${TEST_GPG_KEYID_1}" fi -echo "1..4" +echo "1..6" -# Test pulling from a repo which gives error 500 (internal server error) a lot of the time. +# Sanity check with no network retries and 500s given, pull should fail. +rm ostree-srv httpd repo -rf +setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-500s=99 + +pushd ${test_tmpdir} +ostree_repo_init repo --mode=archive +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo +assert_fail ${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=0 main 2>err.txt +assert_file_has_content err.txt "\(500.*Internal Server Error\)\|\(HTTP 500\)" + +popd +echo "ok no retries after a 500" + +# Test pulling a repo which gives error 500 (internal server error) a lot of the time. +rm ostree-srv httpd repo -rf setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-500s=50 pushd ${test_tmpdir} ostree_repo_init repo --mode=archive ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo -for x in $(seq 200); do - if ${CMD_PREFIX} ostree --repo=repo pull --mirror origin main 2>err.txt; then - echo "Success on iteration ${x}" - break; +for x in $(seq 40); do + if ${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=2 main 2>err.txt; then + echo "Success on iteration ${x}" + break; fi assert_file_has_content err.txt "\(500.*Internal Server Error\)\|\(HTTP 500\)" done @@ -48,6 +62,24 @@ ${CMD_PREFIX} ostree --repo=repo rev-parse main popd echo "ok repeated pull after 500s" +# Test pulling a repo that gives 408s a lot of the time, with many network retries. +rm ostree-srv httpd repo -rf +setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-500s=50 + +pushd ${test_tmpdir} +ostree_repo_init repo --mode=archive +${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo + +# We limit 500s above to 100, so 100 retries should be enough always. +${CMD_PREFIX} ostree --repo=repo pull --mirror origin --network-retries=100 main +echo "Success with big number of network retries" + +${CMD_PREFIX} ostree --repo=repo fsck +${CMD_PREFIX} ostree --repo=repo rev-parse main + +popd +echo "ok big number of retries with one 500s" + # Sanity check with no network retries and 408s given, pull should fail. rm ostree-srv httpd repo -rf setup_fake_remote_repo1 "archive" "${COMMIT_SIGN}" --random-408s=99