OILS / deps / from-apt.sh View on Github | oils.pub

443 lines, 211 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# deps/from-apt.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# These are needed for bootstrapping pip in Python 3.10
11# (Also used by build/py.sh ubuntu-deps)
12#
13# For building Python 3.10 with working 'pip install'
14# libssl-dev: to download packages
15# libffi-dev: for working setuptools
16# zlib1g-dev: needed for 'import zlib'
17declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
18
19# Needed to run unit tests, e.g. 'import readline'
20declare -a PY2_BUILD_DEPS=(libreadline-dev)
21
22# for deps/from-R.sh
23declare -a R_BUILD_DEPS=(
24 r-base-core # R interpreter
25
26 # ICU for the R stringi package. This makes compilation faster; otherwise
27 # it tries to compile it from source.
28 # https://stringi.gagolewski.com/install.html
29 libicu-dev
30)
31
32install-R() {
33 ### For manual use OUTSIDE container
34 apt-install "${R_BUILD_DEPS[@]}"
35}
36
37# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
38
39# TODO: Use this for ALL images
40apt-install() {
41 ### Helper to slim down images
42
43 apt-get install -y --no-install-recommends "$@"
44}
45
46init-deb-cache() {
47 rm -f /etc/apt/apt.conf.d/docker-clean
48 echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
49}
50
51readonly -a BUILD_PACKAGES=(
52 gcc
53 g++ # re2c is C++
54 make # to build re2c
55
56 # for cmark
57 cmake
58
59 # cmake -G Ninja can be used
60 ninja-build
61
62 # For 'deps/wedge.sh unboxed-install'
63 sudo
64
65 # uftrace configure uses pkg-config to find python3 flags
66 pkg-config
67
68 # 2024-06: I think this is necessary for the boxed build of zsh
69 # Not sure why the wedge-setup-debian VM build doesn't need it. Oh probably
70 # because Github Actions pre-installs many packages for you.
71 libncursesw5-dev
72
73 # for USDT probes
74 systemtap-sdt-dev
75
76 "${PY2_BUILD_DEPS[@]}"
77
78 # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
79 # won't work.
80 # TODO: We should move 'pip install' to build time.
81 "${PY3_BUILD_DEPS[@]}"
82
83 # For installing R packages
84 "${R_BUILD_DEPS[@]}"
85)
86
87layer-wedge-bootstrap-debian() {
88 apt-get update
89
90 # Pass additional deps
91 apt-install \
92 "${BUILD_PACKAGES[@]}" \
93 "$@"
94}
95
96layer-wedge-bootstrap-debian-10() {
97 # Default packages
98 layer-wedge-bootstrap-debian
99}
100
101layer-wedge-bootstrap-debian-12() {
102 local -a uftrace_packages=(
103 # uftrace configure detects with #include "Python.h"
104 python3-dev
105 # shared library for uftrace to do dlopen()
106 # requires path in uftrace source
107 libpython3.11
108
109 #libpython3.7
110 )
111
112 layer-wedge-bootstrap-debian "${uftrace_packages[@]}"
113}
114
115readonly PY2=/wedge/oils-for-unix.org/pkg/python2/2.7.18/bin/python
116
117layer-python-symlink() {
118 ### make /usr/bin/python
119
120 # For building CPython in soil-ovm-tarball; done as root
121 ln -s -f -v $PY2 /usr/bin/python
122}
123
124layer-python2-symlink() {
125 # make /usr/bin/python2
126
127 # It's usually better do 'source build/dev-shell.sh' instead ...
128 ln -s -f -v $PY2 /usr/bin/python2
129}
130
131layer-debian-10() {
132 # Can't install packages in Debian without this
133 apt-get update # uses /var/lib/apt
134
135 # uses /var/cache/apt
136 apt-install git python2
137}
138
139layer-debian-12() {
140 # Can't install packages in Debian without this
141 apt-get update # uses /var/lib/apt
142
143 # uses /var/cache/apt
144 # Soil's time-tsv3 can run under python3 too
145 apt-install git python3
146}
147
148layer-locales() {
149 set -x
150 apt-install locales
151
152 # uncomment in a file
153 sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
154 locale-gen --purge en_US.UTF-8
155}
156
157wild() {
158 # for build/py.sh all
159 local -a packages=(
160 gcc # 'cc' for compiling Python extensions
161 g++ # for C++ tarball
162
163 libreadline-dev
164 curl # wait for cpp-tarball
165 ca-certificates # curl https - could do this outside container
166 )
167
168 apt-install "${packages[@]}"
169}
170
171dev-minimal() {
172 local -a packages=(
173 # TODO: remove
174 #python2-dev # for building Python extensions
175 #python-setuptools # Python 2, for flake8
176 #python-pip # flake8 typing
177
178 gcc # for building Python extensions
179 libreadline-dev
180
181 python3-setuptools # mypy
182 python3-pip
183 # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
184 # dep, which recently updated to version 1.5.5
185 python3-dev
186
187 # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
188 busybox-static mksh zsh
189
190 gawk
191
192 # 'ps' used by spec tests
193 procps
194 # for oil-spec task
195 jq
196 )
197
198 apt-install "${packages[@]}"
199}
200
201pea() {
202 # For installing MyPy
203 # apt-install python3-pip
204
205 echo 'None'
206}
207
208other-tests() {
209 local -a packages=(
210 libreadline-dev
211
212 # Compilers for R. TODO: try removing after wedge
213 gcc g++
214
215 make # to build py27.grammar.marshal, ugh
216
217 r-base-core
218 )
219
220 apt-install "${packages[@]}"
221}
222
223cpp-small() {
224 local -a packages=(
225 # for build/py.sh all
226 libreadline-dev
227
228 # To compile Oil
229 g++
230 ninja-build
231
232 # For 32-bit binaries with -m32
233 gcc-multilib
234 g++-multilib
235
236 # For some tests
237 gawk
238
239 # for MyPy git clone https://. TODO: remove when the build is hermetic
240 ca-certificates
241
242 # for test/ltrace
243 ltrace
244
245 # for USDT probes
246 systemtap-sdt-dev
247 )
248
249 apt-install "${packages[@]}"
250}
251
252test-image() {
253 ### For testing builds, not run on CI
254
255 # We don't really need build deps?
256 apt-install build-essential "${PY3_BUILD_DEPS[@]}"
257}
258
259benchmarks() {
260 ### For benchmarks
261
262 local -a packages=(
263 # for build/py.sh all
264 libreadline-dev
265
266 # To build Oils
267 g++
268 ninja-build
269 make # to build R packages
270
271 # to create _test/index.html
272 gawk
273
274 # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
275 valgrind
276
277 # benchmarks compare system shells -- they don't use our spec-bin? In case
278 # there are perf bugs caused by the build
279 busybox-static mksh zsh
280
281 # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
282 wget
283 bzip2 # extracting benchmarks/osh-runtime
284 xz-utils
285
286 # For analyzing benchmarks.
287 r-base-core
288
289 # pgrep used by test/stateful in interactive task
290 # TODO: Could move both Python and C++ to their own image
291 # That will be a good use case once we have
292 procps
293 )
294
295 apt-install "${packages[@]}"
296}
297
298bloaty() {
299 local -a packages=(
300 g++ # for C++ tarball
301 curl # wait for cpp-tarball
302 ca-certificates # curl https - could do this outside container
303 )
304
305 apt-install "${packages[@]}"
306}
307
308benchmarks2() {
309 ### uftrace needs a Python plugin
310
311 local -a packages=(
312 curl # fetch C++ tarball
313 g++ # build it
314
315 # uftrace needs a Python 3 plugin
316 # This is different than 'python3' or 'python3.11' -- it's only the shared
317 # lib?
318 libpython3.11
319
320 # for stable benchmarks.
321 valgrind
322
323 # Analyze uftrace
324 r-base-core
325 )
326
327 apt-install "${packages[@]}"
328}
329
330cpp-spec() {
331 ### For cpp-spec
332
333 local -a packages=(
334 # for build/py.sh all
335 libreadline-dev
336
337 # To build Oil
338 g++
339 ninja-build
340
341 # to create _test/index.html
342 gawk
343
344 # spec tests use these
345 procps
346 jq
347
348 # for MyPy git clone https://. TODO: remove when the build is hermetic
349 ca-certificates
350 )
351
352 apt-install "${packages[@]}"
353}
354
355clang() {
356 ### For cpp-coverage
357
358 local -a packages=(
359 # For build/py.sh minimal
360 libreadline-dev
361 #python2-dev
362
363 # Compile Oils
364 g++
365 ninja-build
366
367 xz-utils # to extract Clang
368
369 # for MyPy git clone https://. TODO: remove when the build is hermetic
370 ca-certificates
371 )
372
373 apt-install "${packages[@]}"
374}
375
376ovm-tarball() {
377 local -a packages=(
378 # build/py.sh all
379 libreadline-dev
380
381 # retrieving spec-bin -- TODO: move to build time
382 wget
383 # for wget https://. TODO: remove when the build is hermetic
384 ca-certificates
385
386 # when spec tests use 'time', dash falls back on 'time' command
387 'time'
388
389 # TODO: probably can remove C++ compiler now that re2c is a wedge
390 gcc
391 g++
392
393 # for cmark
394 cmake
395 # to build Python-2.7.13 (could be a wedge)
396 make
397
398 xz-utils # extract e.g. zsh/yash tarballs
399 bzip2 # extract e.g. busybox tarball
400
401 # for syscall measurements
402 strace
403
404 # used by test/spec-runner.sh
405 gawk
406 )
407
408 apt-install "${packages[@]}"
409}
410
411app-tests() {
412 local -a packages=(
413 # build/py.sh all
414 libreadline-dev
415
416 # retrieving spec-bin -- TODO: move to build time
417 wget
418 # for wget https://. TODO: remove when the build is hermetic
419 ca-certificates
420
421 curl # wait for cpp-tarball
422
423 gcc
424 g++ # for C++ tarball
425
426 # to build ble.sh
427 make
428 # used by ble.sh
429 gawk
430 procps
431
432 # for ble.sh contra
433 libx11-dev
434 libxft-dev
435 libncursesw5-dev
436 )
437
438 apt-install "${packages[@]}"
439}
440
441if test $(basename $0) = 'from-apt.sh'; then
442 "$@"
443fi