OILS / deps / from-apt.sh View on Github | oilshell.org

429 lines, 215 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# deps/from-apt.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# These are needed for bootstrapping pip in Python 3.10
11# (Also used by build/py.sh ubuntu-deps)
12#
13# For building Python 3.10 with working 'pip install'
14# libssl-dev: to download packages
15# libffi-dev: for working setuptools
16# zlib1g-dev: needed for 'import zlib'
17declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
18
19# for deps/from-R.sh
20declare -a R_BUILD_DEPS=(
21 r-base-core # R interpreter
22
23 # ICU for the R stringi package. This makes compilation faster; otherwise
24 # it tries to compile it from source.
25 # https://stringi.gagolewski.com/install.html
26 libicu-dev
27)
28
29install-R() {
30 ### For manual use OUTSIDE container
31 apt-install "${R_BUILD_DEPS[@]}"
32}
33
34# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
35
36# TODO: Use this for ALL images
37apt-install() {
38 ### Helper to slim down images
39
40 apt-get install -y --no-install-recommends "$@"
41}
42
43init-deb-cache() {
44 rm -f /etc/apt/apt.conf.d/docker-clean
45 echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
46}
47
48readonly -a BUILD_PACKAGES=(
49 gcc
50 g++ # re2c is C++
51 make # to build re2c
52
53 # for cmark
54 cmake
55
56 # cmake -G Ninja can be used
57 ninja-build
58
59 # For 'deps/wedge.sh unboxed-install'
60 sudo
61
62 # uftrace configure uses pkg-config to find python3 flags
63 pkg-config
64
65 # 2024-06: I think this is necessary for the boxed build of zsh
66 # Not sure why the wedge-setup-debian VM build doesn't need it. Oh probably
67 # because Github Actions pre-installs many packages for you.
68 libncursesw5-dev
69
70 # for USDT probes
71 systemtap-sdt-dev
72
73 # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
74 # won't work.
75 # TODO: We should move 'pip install' to build time.
76 "${PY3_BUILD_DEPS[@]}"
77
78 # For installing R packages
79 "${R_BUILD_DEPS[@]}"
80)
81
82layer-wedge-bootstrap-debian() {
83 apt-get update
84
85 # Pass aditional deps
86 apt-install "${BUILD_PACKAGES[@]}" "$@"
87}
88
89layer-wedge-bootstrap-debian-10() {
90 # Default packages
91 layer-wedge-bootstrap-debian
92}
93
94layer-wedge-bootstrap-debian-12() {
95 local -a uftrace_packages=(
96 # uftrace configure detects with #include "Python.h"
97 python3-dev
98 # shared library for uftrace to do dlopen()
99 # requires path in uftrace source
100 libpython3.11
101
102 #libpython3.7
103 )
104
105 layer-wedge-bootstrap-debian "${uftrace_packages[@]}"
106}
107
108layer-python-symlink() {
109 ### A special layer for building CPython; done as root
110 ln -s -f -v /usr/bin/python2 /usr/bin/python
111}
112
113layer-debian-10() {
114 # Can't install packages in Debian without this
115 apt-get update # uses /var/lib/apt
116
117 # uses /var/cache/apt
118 apt-install git python2
119}
120
121layer-debian-12() {
122 # Can't install packages in Debian without this
123 apt-get update # uses /var/lib/apt
124
125 # uses /var/cache/apt
126 # Soil's time-tsv3 can run under python3 too
127 apt-install git python3
128}
129
130layer-locales() {
131 apt-install locales
132 # uncomment in a file
133 sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
134 locale-gen --purge en_US.UTF-8
135}
136
137test-image() {
138 ### For testing builds, not run on CI
139
140 apt-install build-essential "${PY3_BUILD_DEPS[@]}"
141}
142
143wild() {
144 # for build/py.sh all
145 local -a packages=(
146 gcc # 'cc' for compiling Python extensions
147 g++ # for C++ tarball
148
149 python2-dev
150 libreadline-dev
151 curl # wait for cpp-tarball
152 ca-certificates # curl https - could do this outside container
153 )
154
155 apt-install "${packages[@]}"
156}
157
158dev-minimal() {
159 local -a packages=(
160 # TODO: remove
161 python2-dev # for building Python extensions
162 python-setuptools # Python 2, for flake8
163 python-pip # flake8 typing
164
165 gcc # for building Python extensions
166 libreadline-dev
167
168 python3-setuptools # mypy
169 python3-pip
170 # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
171 # dep, which recently updated to version 1.5.5
172 python3-dev
173
174 # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
175 busybox-static mksh zsh
176
177 gawk
178
179 # 'ps' used by spec tests
180 procps
181 # for oil-spec task
182 jq
183 )
184
185 apt-install "${packages[@]}"
186}
187
188pea() {
189 # For installing MyPy
190 # apt-install python3-pip
191
192 echo 'None'
193}
194
195other-tests() {
196 local -a packages=(
197 libreadline-dev
198 python2-dev # osh2oil needs build/py.sh minimal
199
200 # Compilers for R. TODO: try removing after wedge
201 gcc g++
202
203 make # to build py27.grammar.marshal, ugh
204
205 r-base-core
206 )
207
208 apt-install "${packages[@]}"
209}
210
211cpp-small() {
212 local -a packages=(
213 # for build/py.sh all
214 libreadline-dev
215 python2-dev
216
217 # To compile Oil
218 g++
219 ninja-build
220
221 # For 32-bit binaries with -m32
222 gcc-multilib
223 g++-multilib
224
225 # For some tests
226 gawk
227
228 # for MyPy git clone https://. TODO: remove when the build is hermetic
229 ca-certificates
230
231 # for test/ltrace
232 ltrace
233
234 # for USDT probes
235 systemtap-sdt-dev
236 )
237
238 apt-install "${packages[@]}"
239}
240
241benchmarks() {
242 ### For benchmarks
243
244 local -a packages=(
245 # for build/py.sh all
246 libreadline-dev
247 python2-dev
248
249 # To build Oils
250 g++
251 ninja-build
252 make # to build R packages
253
254 # to create _test/index.html
255 gawk
256
257 # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
258 valgrind
259
260 # benchmarks compare system shells -- they don't use our spec-bin? In case
261 # there are perf bugs caused by the build
262 busybox-static mksh zsh
263
264 # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
265 wget
266 bzip2 # extracting benchmarks/osh-runtime
267 xz-utils
268
269 # For analyzing benchmarks.
270 r-base-core
271
272 # pgrep used by test/stateful in interactive task
273 # TODO: Could move both Python and C++ to their own image
274 # That will be a good use case once we have
275 procps
276 )
277
278 apt-install "${packages[@]}"
279}
280
281bloaty() {
282 local -a packages=(
283 g++ # for C++ tarball
284 curl # wait for cpp-tarball
285 ca-certificates # curl https - could do this outside container
286 )
287
288 apt-install "${packages[@]}"
289}
290
291benchmarks2() {
292 ### uftrace needs a Python plugin
293
294 local -a packages=(
295 curl # fetch C++ tarball
296 g++ # build it
297
298 # uftrace needs a Python 3 plugin
299 # This is different than 'python3' or 'python3.11' -- it's only the shared
300 # lib?
301 libpython3.11
302
303 # for stable benchmarks.
304 valgrind
305
306 # Analyze uftrace
307 r-base-core
308 )
309
310 apt-install "${packages[@]}"
311}
312
313cpp-spec() {
314 ### For cpp-spec
315
316 local -a packages=(
317 # for build/py.sh all
318 libreadline-dev
319 python2-dev
320
321 # To build Oil
322 g++
323 ninja-build
324
325 # to create _test/index.html
326 gawk
327
328 # spec tests use these
329 procps
330 jq
331
332 # for MyPy git clone https://. TODO: remove when the build is hermetic
333 ca-certificates
334 )
335
336 apt-install "${packages[@]}"
337}
338
339clang() {
340 ### For cpp-coverage
341
342 local -a packages=(
343 # For build/py.sh minimal
344 libreadline-dev
345 python2-dev
346
347 # Compile Oils
348 g++
349 ninja-build
350
351 xz-utils # to extract Clang
352
353 # for MyPy git clone https://. TODO: remove when the build is hermetic
354 ca-certificates
355 )
356
357 apt-install "${packages[@]}"
358}
359
360ovm-tarball() {
361 local -a packages=(
362 # build/py.sh all
363 libreadline-dev
364 python2-dev
365
366 # retrieving spec-bin -- TODO: move to build time
367 wget
368 # for wget https://. TODO: remove when the build is hermetic
369 ca-certificates
370
371 # when spec tests use 'time', dash falls back on 'time' command
372 'time'
373
374 # TODO: probably can remove C++ compiler now that re2c is a wedge
375 gcc
376 g++
377
378 # for cmark
379 cmake
380 # to build Python-2.7.13 (could be a wedge)
381 make
382
383 xz-utils # extract e.g. zsh/yash tarballs
384 bzip2 # extract e.g. busybox tarball
385
386 # for syscall measurements
387 strace
388
389 # used by test/spec-runner.sh
390 gawk
391 )
392
393 apt-install "${packages[@]}"
394}
395
396app-tests() {
397 local -a packages=(
398 # build/py.sh all
399 libreadline-dev
400 python2-dev
401
402 # retrieving spec-bin -- TODO: move to build time
403 wget
404 # for wget https://. TODO: remove when the build is hermetic
405 ca-certificates
406
407 curl # wait for cpp-tarball
408
409 gcc
410 g++ # for C++ tarball
411
412 # to build ble.sh
413 make
414 # used by ble.sh
415 gawk
416 procps
417
418 # for ble.sh contra
419 libx11-dev
420 libxft-dev
421 libncursesw5-dev
422 )
423
424 apt-install "${packages[@]}"
425}
426
427if test $(basename $0) = 'from-apt.sh'; then
428 "$@"
429fi