tests: Add setup for more realistic repo, change pull-many to use

As OSTree has evolved over time, the tests grew with it.  We
didn't start out with static deltas or a summary file, and the
tests reflect this.

What I really want to do is change more of the pull tests, from
corruption/proxying/mirroring etc. to use this more realistic
repo rather than the tiny one the other test creates.

We start by using some of the code from `test-pull-many.sh`, and change that
test to use `--disable-static-deltas` for pull, since the point of that test is
to *not* test deltas.

Still TODO is investigate changing other tests to use this.

Closes: #658
Approved by: jlebon

tests/libtest.sh

@@ -270,6 +270,102 @@ setup_fake_remote_repo1() {
     export OSTREE="${CMD_PREFIX} ostree --repo=repo"
 }
 
+# Set up a large repository for stress testing.
+# Something like the Fedora Atomic Workstation branch which has
+# objects: meta: 7497 content: 103541
+# 9443 directories, 7097 symlinks, 112832 regfiles
+# So we'll make ~11 files per dir, with one of them a symlink
+# Actually, let's cut this down to 1/3 which is still useful.  So:
+# 3147 dirs, with still ~11 files per dir, for 37610 content objects
+setup_exampleos_repo() {
+    args=${1:-}
+    cd ${test_tmpdir}
+    mkdir ostree-srv
+    mkdir -p ostree-srv/exampleos/{repo,build-repo}
+    export ORIGIN_REPO=ostree-srv/exampleos/repo
+    export ORIGIN_BUILD_REPO=ostree-srv/exampleos/build-repo
+    ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} init --mode=archive
+    ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} init --mode=bare-user
+    cd ${test_tmpdir}
+    rm main -rf
+    mkdir main
+    cd main
+    ndirs=3147
+    depth=0
+#    set +x  # No need to spam the logs for this
+    echo "$(date): Generating initial content..."
+    while [ $ndirs -gt 0 ]; do
+        # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up
+        x=$(($ndirs % 3))
+        case $x in
+            0) if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi ;;
+            1|2) if [ $depth -lt 9 ]; then
+                         mkdir dir-${ndirs}
+                         cd dir-${ndirs}
+                         depth=$((depth+1))
+                     else
+                         if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi
+                 fi ;;
+        esac
+        # One symlink - we use somewhat predictable content to have dupes
+        ln -s $(($x % 20)) link-$ndirs
+        # 10 files
+        nfiles=10
+        while [ $nfiles -gt 0 ]; do
+            echo file-$ndirs-$nfiles > f$ndirs-$nfiles
+            nfiles=$((nfiles-1))
+        done
+        ndirs=$((ndirs-1))
+    done
+    cd ${test_tmpdir}
+    set -x
+
+    export REF=exampleos/42/standard
+
+    ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} commit -b ${REF} --tree=dir=main
+    rm main -rf
+    ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} checkout ${REF} main
+
+    find main > files.txt
+    nfiles=$(wc -l files.txt | cut -f 1 -d ' ')
+    # We'll make 5 more commits
+    for iter in $(seq 5); do
+        set +x
+        # Change 10% of files
+        for fiter in $(seq $(($nfiles / 10))); do
+            filenum=$(($RANDOM % ${nfiles}))
+            set +o pipefail
+            filename=$(tail -n +${filenum} < files.txt | head -1)
+            set -o pipefail
+            if test -f $filename; then
+                rm -f $filename
+                echo file-${iter}-${fiter} > ${filename}
+            fi
+        done
+        set -x
+        ${CMD_PREFIX} ostree --repo=${ORIGIN_BUILD_REPO} commit --link-checkout-speedup -b ${REF} --tree=dir=main
+    done
+
+    ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} pull-local --depth=-1 ${ORIGIN_BUILD_REPO}
+
+    for x in "^^" "^" ""; do
+        ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} static-delta generate --from="${REF}${x}^" --to="${REF}${x}"
+    done
+    ${CMD_PREFIX} ostree --repo=${ORIGIN_REPO} summary -u
+
+    cd ${test_tmpdir}/ostree-srv
+    mkdir httpd
+    ${OSTREE_HTTPD} --autoexit --log-file $(pwd)/httpd/httpd.log --daemonize -p httpd/port $args
+    port=$(cat httpd/port)
+    echo "http://127.0.0.1:${port}" > httpd/address
+
+    cd ${test_tmpdir}
+    rm repo -rf
+    ${CMD_PREFIX} ostree --repo=repo init --mode=bare-user
+    ${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat ostree-srv/httpd/address)/exampleos/repo
+    export OSTREE="${CMD_PREFIX} ostree --repo=repo"
+}
+
 setup_os_boot_syslinux() {
     # Stub syslinux configuration
     mkdir -p sysroot/boot/loader.0

tests/test-pull-many.sh

@@ -21,80 +21,17 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
-setup_fake_remote_repo1 "archive-z2"
+setup_exampleos_repo
-cd ${test_tmpdir}
-rm ostree-srv/gnomerepo/ -rf
-mkdir ostree-srv/gnomerepo/
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo init --mode=archive
 
 echo '1..1'
 
-# Simulate a large archive pull from scratch.  Large here is
-# something like the Fedora Atomic Workstation branch which has
-# objects: meta: 7497 content: 103541
-# 9443 directories, 7097 symlinks, 112832 regfiles
-# So we'll make ~11 files per dir, with one of them a symlink
-
 cd ${test_tmpdir}
-rm main -rf
-mkdir main
-cd main
-ndirs=9443
-depth=0
-echo "$(date): Generating content..."
-set +x  # No need to spam the logs for this
-while [ $ndirs -gt 0 ]; do
-    # 2/3 of the time, recurse a dir, up to a max of 9, otherwise back up
-    x=$(($ndirs % 3))
-    case $x in
-        0) if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi ;;
-        1|2) if [ $depth -lt 9 ]; then
-                 mkdir dir-${ndirs}
-                 cd dir-${ndirs}
-                 depth=$((depth+1))
-             else
-                 if [ $depth -gt 0 ]; then cd ..; depth=$((depth-1)); fi
-             fi ;;
-    esac
-
-    # One symlink - we use somewhat predictable content to have dupes
-    ln -s $(($x % 20)) link-$ndirs
-    # 10 files
-    nfiles=10
-    while [ $nfiles -gt 0 ]; do
-        echo file-$ndirs-$nfiles > f$ndirs-$nfiles
-        nfiles=$((nfiles-1))
-    done
-    ndirs=$((ndirs-1))
-done
 set -x
-cd ${test_tmpdir}
-mkdir build-repo
-${CMD_PREFIX} ostree --repo=build-repo init --mode=bare-user
-echo "$(date): Committing content..."
-${CMD_PREFIX} ostree --repo=build-repo commit -b main -s 'big!' --tree=dir=main
-for x in commit dirtree dirmeta file; do
-    find build-repo/objects -name '*.'${x} |wc -l > ${x}count
-    echo "$x: " $(cat ${x}count)
-done
-assert_file_has_content commitcount '^1$'
-assert_file_has_content dirmetacount '^1$'
-assert_file_has_content filecount '^94433$'
-${CMD_PREFIX} ostree --repo=ostree-srv/gnomerepo pull-local build-repo
 
 echo "$(date): Pulling content..."
-rm repo -rf
+rev=$(${CMD_PREFIX} ostree --repo=ostree-srv/exampleos/repo rev-parse ${REF})
-${CMD_PREFIX} ostree --repo=repo init --mode=archive
+${CMD_PREFIX} ostree --repo=repo pull --disable-static-deltas --mirror origin ${REF}
-${CMD_PREFIX} ostree --repo=repo remote add --set=gpg-verify=false origin $(cat httpd-address)/ostree/gnomerepo
-
-${CMD_PREFIX} ostree --repo=repo pull --mirror origin main
 ${CMD_PREFIX} ostree --repo=repo fsck
-for x in commit dirtree dirmeta filez; do
+assert_streq ${rev} $(${CMD_PREFIX} ostree --repo=repo rev-parse ${REF})
-    find repo/objects -name '*.'${x} |wc -l > ${x}count
-    echo "$x: " $(cat ${x}count)
-done
-assert_file_has_content commitcount '^1$'
-assert_file_has_content dirmetacount '^1$'
-assert_file_has_content filezcount '^94433$'
 
 echo "ok"