OILS / soil / wait.sh View on Github | oilshell.org

142 lines, 69 significant
1#!/usr/bin/env bash
2#
3# Wait for artifacts on a web server.
4#
5# Usage:
6# soil/wait.sh
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13source soil/common.sh
14
15fast-curl-until-200() {
16 ### Retry fetch until HTTP 200, REUSING curl process AND connection
17
18 # Similar to
19 # https://stackoverflow.com/questions/42873285/curl-retry-mechanism
20 # --retry-all-errors is 7.71 !
21
22 # --retry-all-errors not present in curl 7.58.0 on Ubuntu 18.04
23 # Do we have to upgrade? We're using Debian buster-slim
24 #
25 # curl 7.64 ! Gah.
26
27 # Curl versions
28 #
29 # 7.58 - Ubuntu 18.04, --retry-all-errors not present
30 # 7.64 - Debian Buster slim, our container base
31 # https://packages.debian.org/buster/curl
32 # 7.71 - --retry-all-errors
33 # 7.88 - Debian Bookworm
34
35 local url=$1
36 local out_path=$2
37 local num_retries=${3:-10} # number of times through the loop
38 local interval=${4:-10} # retry every n seconds
39
40 mkdir -p "$(dirname $out_path)"
41
42 # --retry-all-errors and --fail are used to make curl try on a 404
43
44 # Note: might need --retry-conn-refused as well, in case the server
45 # disconnects
46
47 curl \
48 --output $out_path \
49 --max-time 10 \
50 --retry $num_retries \
51 --retry-all-errors \
52 --fail \
53 --retry-delay $interval \
54 $url
55}
56
57curl-until-200() {
58 ### bash version of the function above
59
60 local url=$1
61 local out_path=$2
62 local num_retries=${3:-10} # number of times through the loop
63 local interval=${4:-10} # retry every n seconds
64
65 mkdir -p "$(dirname $out_path)"
66
67 local i=0
68 while true; do
69 local http_code
70 http_code=$(curl --output $out_path --write-out '%{http_code}' $url)
71
72 if test "$http_code" = 200; then
73 log "Curl wrote $out_path"
74 ls -l $out_path
75 break;
76 fi
77
78 log "HTTP status $http_code; retrying in $interval seconds"
79 sleep $interval
80
81 i=$(( i + 1 ))
82 if test $i -eq $num_retries; then
83 log "Giving up after $num_retries tries"
84 return 1 # fail
85 fi
86 done
87}
88
89# Users
90# - test/wild.sh soil-run
91# - benchmarks/perf.sh in the raw-vm task
92# - test/ble.sh, in app-tests task
93
94for-cpp-tarball() {
95 local prefix=${1:-github-}
96
97 # There are now 3 tasks waiting for the cpp-tarball
98 # wild - might want to wait 80 seconds
99 # app-tests - run clone, build, bash first
100 # raw-vm - install perf first
101 #
102 # So just wait 1 second
103 local sleep_secs=${2:-1}
104
105 # Retry for 18 times, every 10 seconds = 3 minutes.
106
107 # If we have 10 clients, then we're hitting it once a second, which is not
108 # unreasonable. We're also keeping 10 connections
109
110 local num_retries=${3:-18}
111 local interval=${4:-10}
112
113 local git_commit_url
114 git_commit_url=$(git-commit-url $prefix)
115
116 local url="$git_commit_url/oils-for-unix.tar"
117
118 set -x
119 sleep $sleep_secs
120
121 curl-until-200 $url _release/oils-for-unix.tar $num_retries $interval
122}
123
124readonly TEST_FILE='oilshell.org/tmp/curl-test'
125
126for-test-file() {
127 curl-until-200 "https://www.$TEST_FILE" _tmp/$(basename $TEST_FILE) 5 10
128}
129
130touch-remote() {
131 ssh oilshell.org "echo hi > $TEST_FILE"
132}
133
134rm-remote() {
135 ssh oilshell.org "rm -v $TEST_FILE"
136}
137
138test-for-cpp-tarball() {
139 for-cpp-tarball
140}
141
142"$@"