OILS / test / spec-compat.sh View on Github | oils.pub

303 lines, 149 significant
1#!/usr/bin/env bash
2#
3# Test OSH against any shell
4#
5# Usage:
6# test/spec-compat.sh <function name>
7
8: ${LIB_OSH=stdlib/osh}
9source $LIB_OSH/bash-strict.sh
10source $LIB_OSH/task-five.sh
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13
14source build/dev-shell.sh # put mksh etc. in $PATH
15source test/common.sh
16source test/spec-common.sh
17
18OSH_TARGET=_bin/cxx-asan/osh
19OSH=$PWD/$OSH_TARGET
20
21# To compare against:
22# - toysh
23# - brush
24# - rusty_bash
25# - ksh93 - Debian package
26
27# Metrics
28# - binary size - stripped
29# - lines of source code - I think we get this from DWARF debug info
30# - https://claude.ai/chat/40597e2e-4d1e-42b4-a756-7a265f01cc5a shows options
31# - llvm-dwarfdump
32# - Python lib https://github.com/eliben/pyelftools/
33# - right now this isn't worth it - spec tests are more important
34# - unsafe functions / methods?
35# - cargo geiger is also hard to parse
36
37readonly TOYBOX_DIR=~/src/toybox-0.8.12
38
39readonly SUSH_DIR=../../shells/rusty_bash
40readonly BRUSH_DIR=../../shells/brush
41
42readonly SUSH=$PWD/$SUSH_DIR/target/release/sush
43readonly BRUSH=$PWD/$BRUSH_DIR/target/release/brush
44
45# these are all roughly ksh compatible
46readonly -a SHELLS=(bash dash ash zsh mksh ksh $TOYBOX_DIR/sh $SUSH $BRUSH $OSH)
47
48download-toybox() {
49 #mkdir -p ~/src
50 wget --directory ~/src --no-clobber \
51 https://landley.net/toybox/downloads/toybox-0.8.12.tar.gz
52}
53
54build-toybox() {
55 pushd $TOYBOX_DIR
56
57 make toybox
58 # warning: using unfinished code
59 make sh
60
61 popd
62}
63
64update-rust() {
65 . ~/.cargo/env
66 time rustup update
67}
68
69build-brush() {
70 local pull=${1:-}
71
72 pushd ../../shells/brush
73
74 if test -n "$pull"; then
75 git pull
76 fi
77
78 . ~/.cargo/env
79
80 # Test incremental build speed
81 # - debug: 3.8 seconds
82 # - release: 1:06 minutes !
83 # touch brush-core/src/shell.rs
84
85 # 41s
86 time cargo build
87 echo
88
89 # 1m 49s
90 # It builds a stripped binary by default - disable that for metrics
91 RUSTFLAGS='-C strip=none' time cargo build --release
92 echo
93
94 popd
95}
96
97build-sush() {
98 local pull=${1:-}
99
100 pushd ../../shells/rusty_bash
101
102 if test -n "$pull"; then
103 git pull
104 fi
105
106 . ~/.cargo/env
107
108 # Test incremental build speed
109 # - debug: 1 second
110 # - release: 6 seconds
111 #touch src/core.rs
112
113 # 10 seconds
114 time cargo build
115 echo
116
117 # 15 seconds
118 time cargo build --release
119 echo
120
121 popd
122}
123
124binary-sizes() {
125 local oils=_bin/cxx-opt/bin/oils_for_unix.mycpp.stripped
126 ninja $oils
127
128 pushd $BRUSH_DIR
129 local out=target/release/brush.stripped
130 strip -o $out target/release/brush
131 local brush=$BRUSH_DIR/$out
132 popd
133
134 pushd $SUSH_DIR
135 local out=target/release/sush.stripped
136 strip -o $out target/release/sush
137 local sush=$SUSH_DIR/$out
138 popd
139
140 echo
141 ls -l --si $oils $brush $sush $TOYBOX_DIR/sh
142
143 # These aren't dynamically linked to GNU readline, or libstdc++
144 echo
145 ldd $oils $brush $sush $TOYBOX_DIR/sh
146}
147
148symbols() {
149 pushd ../../shells/brush
150 #file target/release/brush
151
152 echo 'BRUSH'
153 # 6272
154 nm target/release/brush | wc -l
155 popd
156
157 pushd ../../shells/rusty_bash
158 # Not stripped
159 #file target/release/sush
160
161 echo 'SUSH'
162 # 4413
163 nm target/release/sush | wc -l
164 # More symbols
165 # nm target/debug/sush | wc -l
166 popd
167
168 #local osh=_bin/cxx-opt/bin/oils_for_unix.mycpp.stripped
169 local osh=_bin/cxx-opt/bin/oils_for_unix.mycpp
170 local dbg=_bin/cxx-dbg/bin/oils_for_unix.mycpp
171 ninja $osh
172
173 echo 'OSH'
174 # 9857 - lots of string literals?
175 nm $osh | wc -l
176 #nm $osh | less
177
178 #ninja $dbg
179 # 17570
180 #nm $dbg | wc -l
181}
182
183install-geiger() {
184 # https://github.com/geiger-rs/cargo-geiger
185 . ~/.cargo/env
186
187 # 2:34 minutes
188 cargo install --locked cargo-geiger
189}
190
191# This is DESTRUCTIVE
192geiger-report() {
193 if true; then
194 pushd ../../shells/brush
195
196 . ~/.cargo/env
197
198 # doesn't work
199 #time cargo geiger --workspace
200 #time cargo geiger --package brush-core --package brush-parser
201
202 popd
203 fi
204
205 if false; then
206 pushd ../../shells/rusty_bash
207
208 . ~/.cargo/env
209
210 # this cleans the build
211 #
212 # Functions Expressions Impls Traits Methods
213 # 181/1056 9377/45040 114/158 30/32 463/2887
214 #
215 # x/y
216 # x = unsafe used by build
217 # y = unsafe in crate
218
219 # ~7 seconds
220 time cargo geiger
221
222 popd
223 fi
224}
225
226#
227# Spec Tests
228#
229
230run-file() {
231 local spec_name=${1:-smoke}
232 shift # Pass list of shells
233
234 local spec_subdir='compat'
235 local base_dir=_tmp/spec/$spec_subdir
236 mkdir -v -p $base_dir
237
238 # spec/tilde hangs under toysh - need timeout
239 sh-spec spec/$spec_name.test.sh \
240 --tsv-output $base_dir/${spec_name}.result.tsv \
241 --timeout 1 \
242 "$@" \
243 "${SHELLS[@]}"
244}
245
246osh-all() {
247 # Since we're publishing these, make sure we start with a clean slate
248 rm -r -f -v _tmp/spec
249
250 ninja $OSH_TARGET
251
252 test/spec-runner.sh shell-sanity-check "${SHELLS[@]}"
253
254 local spec_subdir=compat
255
256 local status
257 set +o errexit
258 # $suite $compare_mode
259 test/spec-runner.sh all-parallel \
260 compat spec-compat $spec_subdir "$@"
261 status=$?
262 set -o errexit
263
264 # Write comparison even if we failed
265 test/spec-compat-html.sh write-compare-html $spec_subdir
266
267 return $status
268}
269
270#
271# Misc
272#
273
274list() {
275 mkdir -p _tmp/spec # _all-parallel also does this
276 test/spec-runner.sh write-suite-manifests
277 wc -l _tmp/spec/SUITE-*
278
279 # TODO:
280 # - Remove zsh test files?
281 # - What about *-bash test cases? These aren't clearly organized
282
283 cat _tmp/spec/SUITE-osh.txt
284}
285
286readonly ERRORS=(
287 'echo )' # parse error
288 'cd -z' # usage error
289 'cd /zzz' # runtime error
290)
291
292survey-errors() {
293 set +o errexit
294 for sh in "${SHELLS[@]}"; do
295 echo
296 echo " === $sh"
297 for code in "${ERRORS[@]}"; do
298 $sh -c "$code"
299 done
300 done
301}
302
303task-five "$@"