OILS / deps / wedge.sh View on Github | oilshell.org

517 lines, 273 significant
1#!/usr/bin/env bash
2#
3# Build a wedge.
4#
5# Usage:
6# deps/wedge.sh <function name>
7#
8# Containerized build:
9#
10# $0 build deps/source.medo/re2c/
11#
12# Host build, without containers:
13#
14# $0 unboxed deps/source.medo/re2c/
15#
16# Individual steps:
17#
18# $0 unboxed-make deps/source.medo/re2c/
19# $0 unboxed-install deps/source.medo/re2c/
20# $0 unboxed-smoke-test deps/source.medo/re2c/
21#
22# Host dir structure:
23#
24# ~/git/oilshell/oil
25# deps/
26# source.medo/ # Source Files
27# MEDO # points to silo
28# re2c.wedge.sh # later it will be re2c.wedge.hay
29# re2c-3.0.blob # .tar.gz file that you can 'medo sync'
30# re2c-3.1.blob
31# opaque.medo/ # Binary files, e.g. Clang
32# derived.medo/ # Saved output of 'wedge build'
33#
34# _build/ # Temp dirs and output
35# obj/ # for C++ / Ninja
36# deps-source/ # sync'd from deps/source.medo - should it be
37# # _build/wedge/source?
38# wedge/ # for containerized builds
39# tmp/ # build directory
40# boxed/ # output of containerized build
41# # TODO: rename from /binary/
42# logs/
43# smoke-test/ # current dir for smoke test
44
45# Every package ("wedge") has these dirs associated with it:
46#
47# 1. Dir with additional tests / files, near tarball and *.wedge.sh ($wedge_dir)
48# 2. Where it's extracted ($src_dir)
49# 3. The temp dir where you run ./configure --prefix; make; make install ($build_dir)
50# 4. The dir to install to ($install_dir)
51# 5. The temp dir where the smoke test is run
52
53# For Debian/Ubuntu
54
55# Note: xz-utils needed to extract, but medo should make that transparent?
56#
57# Container dir structure
58#
59# /home/uke/
60# tmp-mount/
61# _build/ # Build into this temp dir
62# deps-source/
63# re2c/
64# re2c-3.0.tar.xz
65# re2c-3.0/ # Extract it here
66# wedge/
67# re2c
68# /wedge/ # Output is mounted to oil/_mount/wedge-out
69# oilshell.org/
70# pkg/
71# re2c/
72# 3.0/
73# debug-info/ # Probably needs to be at an absolute path because of
74# # --debug-link
75# re2c/
76# 3.0/
77#
78# Then Dockerfile.wild does:
79#
80# COPY _build/wedge/binary/oils-for-unix.org/pkg/re2c/3.0 \
81# /wedge/oils-for-unix.org/pkg/re2c/3.0
82
83set -o nounset
84set -o pipefail
85set -o errexit
86
87REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
88readonly REPO_ROOT
89
90OILS_ABSOLUTE_ROOT='/wedge/oils-for-unix.org'
91
92# The user may build a wedge outside a container here
93OILS_RELATIVE_ROOT="$HOME/wedge/oils-for-unix.org"
94
95log() {
96 echo "$@" >&2
97}
98
99die() {
100 log "$0: fatal: $@"
101 exit 1
102}
103
104#
105# Dirs
106#
107
108source-dir() {
109 if test -n "${WEDGE_TARBALL_NAME:-}"; then
110 # for Python-3.10.4 to override 'python3' package name
111 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_TARBALL_NAME-$WEDGE_VERSION"
112
113 else
114 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_NAME-$WEDGE_VERSION"
115 fi
116}
117
118build-dir() {
119 # call it tmp-build?
120 echo "$REPO_ROOT/_build/wedge/tmp/$WEDGE_NAME-$WEDGE_VERSION"
121}
122
123install-dir() {
124 local prefix
125 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
126 prefix=$OILS_ABSOLUTE_ROOT
127 else
128 prefix=$OILS_RELATIVE_ROOT
129 fi
130
131 # TODO: We want to support multiple versions of the same wedge
132 # So maybe we can provide
133 #
134 # WEDGE_VERSION_LIST='4.4 5.2'
135 #
136 # And then provide a flag to select them?
137
138 echo "$prefix/pkg/$WEDGE_NAME/$WEDGE_VERSION"
139}
140
141smoke-test-dir() {
142 echo "$REPO_ROOT/_build/wedge/smoke-test/$WEDGE_NAME-$WEDGE_VERSION"
143}
144
145load-wedge() {
146 ### source .wedge.sh file and ensure it conforms to protocol
147
148 local wedge_dir=$1
149 local version_requested=${2:-}
150
151 echo "Loading $wedge_dir"
152 echo
153
154 source $wedge_dir/WEDGE
155
156 echo " OK name: ${WEDGE_NAME?"$wedge_dir: WEDGE_NAME required"}"
157
158 # This WEDGE supports a single version.
159 if test -n "${WEDGE_VERSION:-}"; then
160 echo " -- single version: $WEDGE_VERSION"
161 fi
162
163 # Can validate version against this
164 if test -n "${WEDGE_VERSION_LIST:-}"; then
165 echo " -- version list: $WEDGE_VERSION_LIST"
166
167 if test -z "$version_requested"; then
168 die "FAIL Expected explicit version, one of: $WEDGE_VERSION_LIST"
169 fi
170
171 case "$WEDGE_VERSION_LIST" in
172 *"$version_requested"*)
173 echo " OK Setting WEDGE_VERSION to $version_requested"
174 WEDGE_VERSION=$version_requested
175 ;;
176 *)
177 die "FAIL Requested version $version_requested should be one of: $WEDGE_VERSION_LIST"
178 ;;
179 esac
180 fi
181
182 if test -n "${WEDGE_TARBALL_NAME:-}"; then
183 echo " -- tarball name: $WEDGE_TARBALL_NAME"
184 fi
185 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
186 echo ' -- WEDGE_IS_ABSOLUTE'
187 fi
188
189 # Python and R installation use the network
190 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
191 echo ' -- WEDGE_LEAKY_BUILD'
192 fi
193
194 if declare -f wedge-make; then
195 echo " OK wedge-make"
196 elif declare -f wedge-make-from-source-dir; then
197 echo " OK wedge-make-from-source-dir"
198 else
199 die "$wedge_dir: wedge-make(-from-source-dir) not declared"
200 fi
201
202 if declare -f wedge-install; then
203 echo " OK wedge-install"
204 elif declare -f wedge-make-from-source-dir; then
205 echo " OK wedge-install-from-source-dir"
206 else
207 die "$wedge_dir: wedge-install(-from-source-dir) not declared"
208 fi
209
210 # Just one function for now
211 for func in wedge-smoke-test; do
212 if declare -f $func > /dev/null; then
213 echo " OK $func"
214 else
215 die "$wedge_dir: $func not declared"
216 fi
217 done
218 echo
219
220 echo "Loaded $wedge_dir"
221 echo
222}
223
224_run-sourced-func() {
225 "$@"
226}
227
228#
229# Actions
230#
231
232validate() {
233 local wedge=$1
234 local version_requested=${2:-}
235
236 load-wedge $wedge "$version_requested"
237}
238
239unboxed-make() {
240 ### Build on the host
241
242 local wedge=$1 # e.g. re2c.wedge.sh
243 local version_requested=${2:-} # e.g. 5.2
244
245 load-wedge $wedge "$version_requested"
246
247 local source_dir
248 source_dir=$(source-dir)
249 echo " SRC $source_dir"
250
251 local build_dir
252 build_dir=$(build-dir)
253
254 # NOT created because it might require root permissions!
255 local install_dir
256 install_dir=$(install-dir)
257
258 rm -r -f -v $build_dir
259 mkdir -p $build_dir
260
261 if declare -f wedge-make-from-source-dir; then
262 # e.g. for yash, which can't build outside the source tree
263 pushd $source_dir
264 wedge-make-from-source-dir $source_dir $install_dir
265 popd
266 else
267 pushd $build_dir
268 wedge-make $source_dir $build_dir $install_dir
269 popd
270 fi
271}
272
273
274# https://www.gnu.org/prep/standards/html_node/Standard-Targets.html
275
276# Do not strip executables when installing them. This helps eventual
277# debugging that may be needed later, and nowadays disk space is cheap and
278# dynamic loaders typically ensure debug sections are not loaded during
279# normal execution. Users that need stripped binaries may invoke the
280# install-strip target to do that.
281
282_unboxed-install() {
283 local wedge=$1 # e.g. re2c.wedge.sh
284 local version_requested=${2:-} # e.g. 5.2
285
286 load-wedge $wedge "$version_requested"
287
288 local source_dir
289 source_dir=$(source-dir)
290
291 local build_dir
292 build_dir=$(build-dir)
293
294 local install_dir
295 install_dir=$(install-dir)
296 mkdir -p $install_dir
297
298 if declare -f wedge-make-from-source-dir; then
299 pushd $source_dir
300 wedge-install-from-source-dir $source_dir $install_dir
301 popd
302 else
303 # Note: install-dir needed for time-helper, but not others
304 #
305 # I think it would nicer to pushd $build_dir in most cases
306
307 wedge-install $build_dir $install_dir
308 fi
309}
310
311unboxed-install() {
312 local wedge=$1 # e.g. re2.wedge.sh
313
314 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
315 sudo $0 _unboxed-install "$@"
316 else
317 _unboxed-install "$@"
318 fi
319}
320
321unboxed-smoke-test() {
322 local wedge_dir=$1 # e.g. re2c/ with WEDGE
323 local version_requested=${2:-} # e.g. 5.2
324
325 load-wedge $wedge_dir "$version_requested"
326
327 local smoke_test_dir
328 smoke_test_dir=$(smoke-test-dir)
329 local install_dir
330 install_dir=$(install-dir)
331
332 echo ' SMOKE TEST'
333
334 local abs_wedge_dir
335 case $wedge_dir in
336 /*) # it's already absolute
337 abs_wedge_dir=$wedge_dir
338 ;;
339 *)
340 abs_wedge_dir=$PWD/$wedge_dir
341 ;;
342 esac
343
344 # TODO: To ensure a clean dir, it might be better to test that it does NOT
345 # exist first, and just make it. If it exists, then remove everything.
346
347 rm -r -f -v $smoke_test_dir
348 mkdir -p $smoke_test_dir
349
350 pushd $smoke_test_dir
351 set -x
352 wedge-smoke-test $install_dir $abs_wedge_dir
353 set +x
354 popd
355
356 echo ' OK'
357}
358
359unboxed-stats() {
360 local wedge=$1
361
362 load-wedge $wedge "$version_requested"
363
364 du --si -s $(source-dir)
365 echo
366
367 du --si -s $(build-dir)
368 echo
369
370 du --si -s $(install-dir)
371 echo
372}
373
374unboxed() {
375 local wedge_dir=$1
376
377 # Can override default version. Could be a flag since it's optional? But
378 # right now we always pass it.
379 local version_requested=${2:-}
380
381 # TODO:
382 # - Would be nice to export the logs somewhere
383
384 unboxed-make $wedge_dir "$version_requested"
385
386 unboxed-install $wedge_dir "$version_requested"
387
388 unboxed-smoke-test $wedge_dir "$version_requested"
389}
390
391readonly DEFAULT_DISTRO=debian-10 # Debian Buster
392
393DOCKER=${DOCKER:-docker}
394
395boxed() {
396 ### Build inside a container, and put output in a specific place.
397
398 # TODO: Specify the container OS, CPU like x86-64, etc.
399
400 local wedge=$1
401 local version_requested=${2:-}
402 local distro=${3:-$DEFAULT_DISTRO}
403
404 local bootstrap_image=oilshell/wedge-bootstrap-$distro
405
406 load-wedge $wedge "$version_requested"
407
408 # Permissions will be different, so we separate the two
409
410 local wedge_host_dir
411 local wedge_guest_dir
412 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
413 wedge_host_dir=_build/wedge/binary # TODO: rename to /absolute/
414 wedge_guest_dir=/wedge
415 else
416 wedge_host_dir=_build/wedge/relative
417 wedge_guest_dir=/home/uke0/wedge
418 fi
419
420 mkdir -v -p $wedge_host_dir
421
422 # TODO:
423 #
424 # Mount
425 # INPUTS: the PKG.wedge.sh, and the tarball
426 # CODE: this script: deps/wedge.sh
427 # OUTPUT: /wedge/oils-for-unix.org
428 # TODO: Also put logs and symbols somewhere
429
430 # Run unboxed-{build,install,smoke-test} INSIDE the container
431 local -a args=(
432 sh -c 'cd ~/oil; deps/wedge.sh unboxed "$1" "$2"'
433 dummy "$wedge" "$version_requested"
434 )
435
436 local -a docker_flags=()
437 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
438 :
439 else
440 # Disable network for hermetic builds. TODO: Add automated test
441 docker_flags=( --network none )
442 fi
443
444 # TODO:
445 # - Don't mount the whole REPO_ROOT
446 # - We want the bare minimum of files, for cache invalidation
447 # - Maybe make it read only
448 # - Bind mount WEDGE_DEPS='', space separated list of paths
449 # - py3-libs depends on python3 and mypy wedges!
450
451 # -E to preserve CONTAINERS_REGISTRIES_CONF
452 sudo -E $DOCKER run "${docker_flags[@]}" \
453 --mount "type=bind,source=$REPO_ROOT,target=/home/uke0/oil" \
454 --mount "type=bind,source=$PWD/$wedge_host_dir,target=$wedge_guest_dir" \
455 $bootstrap_image \
456 "${args[@]}"
457}
458
459smoke-test() {
460 local wedge_dir=$1
461 local wedge_out_dir=${2:-_build/wedge/binary} # TODO: rename to /boxed
462 local version_requested=${3:-}
463 local distro=${4:-$DEFAULT_DISTRO}
464 local debug_shell=${5:-}
465
466 load-wedge $wedge_dir "$version_requested"
467
468 local bootstrap_image=oilshell/wedge-bootstrap-$distro
469
470 local -a args=(
471 sh -c 'cd ~/oil; deps/wedge.sh unboxed-smoke-test $1' dummy "$wedge_dir"
472 )
473 local -a docker_flags=()
474 if test -n "$debug_shell"; then
475 docker_flags=( -i -t )
476 args=( "$debug_shell" )
477 fi
478
479 local wedge_mount_dir
480 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
481 wedge_mount_dir=/wedge
482 else
483 wedge_mount_dir=/home/uke0/wedge
484 fi
485
486 sudo $DOCKER run "${docker_flags[@]}" \
487 --network none \
488 --mount "type=bind,source=$REPO_ROOT,target=/home/uke0/oil" \
489 --mount "type=bind,source=$PWD/$wedge_out_dir,target=$wedge_mount_dir" \
490 $bootstrap_image \
491 "${args[@]}"
492}
493
494if [[ $# -eq 0 || $1 =~ ^(--help|-h)$ ]]; then
495 # A trick for help. TODO: Move this to a common file, and combine with help
496 # in test/spec.sh.
497
498 awk '
499 $0 ~ /^#/ { print $0 }
500 $0 !~ /^#/ { print ""; exit }
501 ' $0
502 exit
503fi
504
505case $1 in
506 validate|\
507 unboxed|\
508 unboxed-make|unboxed-install|_unboxed-install|\
509 unboxed-smoke-test|unboxed-stats|\
510 boxed|smoke-test)
511 "$@"
512 ;;
513
514 *)
515 die "$Invalid action '$1'"
516 ;;
517esac