OILS / test / spec-compat-html.sh View on Github | oils.pub

402 lines, 92 significant
1#!/usr/bin/env bash
2#
3# Create reports, published at https://pages.oils.pub/
4#
5# Usage:
6# ./spec-compat-html.sh <function name>
7#
8# TODO:
9# - Deploy HTML
10# - add tree html
11# - improve pages.oils.pub/ index.html
12# - Epoch or Build timestamp on page
13# - Improve
14# - summary/percentages in TOP.html?
15# - More shells: might as well include ash, dash, ysh
16# - Make a container with `buildah`, so others can collaborate?
17# - Refactor
18# - maybe clean up test/spec-runner.sh arguments
19
20: ${LIB_OSH=stdlib/osh}
21source $LIB_OSH/bash-strict.sh
22source $LIB_OSH/task-five.sh
23
24REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
25
26source benchmarks/common.sh # cmark function
27source test/common.sh
28source test/spec-common.sh
29source test/tsv-lib.sh # tsv-row
30source web/table/html.sh # table-sort-begin
31
32# Matches SHELLS in test/spec-compat.sh
33readonly -a SH_LABELS=(bash dash ash zsh mksh ksh toysh sush brush osh)
34
35summary-tsv-row() {
36 ### Print one row or the last total row
37
38 local report=$1
39 local spec_subdir=$2
40 shift 2
41
42 if test $# -eq 1; then
43 local spec_name=$1
44 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.result.tsv )
45 else
46 local spec_name='TOTAL'
47 local -a tsv_files=( "$@" )
48 fi
49
50 awk -v report=$report -v spec_name=$spec_name '
51# skip the first row
52FNR != 1 {
53 case_num = $1
54 sh = $2
55 result = $3
56
57 if (sh == "bash") {
58 bash[result] += 1
59 } else if (sh == "dash") {
60 dash[result] += 1
61 } else if (sh == "ash") {
62 ash[result] += 1
63 } else if (sh == "zsh") {
64 zsh[result] += 1
65 } else if (sh == "mksh") {
66 mksh[result] += 1
67 } else if (sh == "ksh") {
68 ksh[result] += 1
69 } else if (sh == "toysh") {
70 toysh[result] += 1
71 } else if (sh == "sush") {
72 sush[result] += 1
73 } else if (sh == "brush") {
74 brush[result] += 1
75 } else if (sh == "osh") {
76 osh[result] += 1
77 }
78}
79
80END {
81 if (spec_name == "TOTAL") {
82 href = ""
83 } else {
84 href = sprintf("%s.html", spec_name)
85 }
86
87 if (report == "PASSING") {
88 bash_total = ("pass" in bash) ? bash["pass"] : 0
89 dash_total = ("pass" in dash) ? dash["pass"] : 0
90 ash_total = ("pass" in ash) ? ash["pass"] : 0
91 zsh_total = ("pass" in zsh) ? zsh["pass"] : 0
92 mksh_total = ("pass" in mksh) ? mksh["pass"] : 0
93 ksh_total = ("pass" in ksh) ? ksh["pass"] : 0
94 toysh_total = ("pass" in toysh) ? toysh["pass"] : 0
95 sush_total = ("pass" in sush) ? sush["pass"] : 0
96 brush_total = ("pass" in brush) ? brush["pass"] : 0
97 osh_total = ("pass" in osh) ? osh["pass"] : 0
98
99 } else if (report == "DELTA-osh") {
100 bash_total = bash["pass"] - osh["pass"]
101 dash_total = dash["pass"] - osh["pass"]
102 ash_total = ash["pass"] - osh["pass"]
103 zsh_total = zsh["pass"] - osh["pass"]
104 mksh_total = mksh["pass"] - osh["pass"]
105 ksh_total = ksh["pass"] - osh["pass"]
106 toysh_total = toysh["pass"] - osh["pass"]
107 sush_total = sush["pass"] - osh["pass"]
108 brush_total = brush["pass"] - osh["pass"]
109 osh_total = osh["pass"] - osh["pass"]
110
111 } else if (report == "DELTA-bash") {
112 bash_total = bash["pass"] - bash["pass"]
113 dash_total = dash["pass"] - bash["pass"]
114 ash_total = ash["pass"] - bash["pass"]
115 zsh_total = zsh["pass"] - bash["pass"]
116 mksh_total = mksh["pass"] - bash["pass"]
117 ksh_total = ksh["pass"] - bash["pass"]
118 toysh_total = toysh["pass"] - bash["pass"]
119 sush_total = sush["pass"] - bash["pass"]
120 brush_total = brush["pass"] - bash["pass"]
121 osh_total = osh["pass"] - bash["pass"]
122 }
123
124 # TODO: change this color
125 row_css_class = "cpp-good" # green
126
127 row = sprintf("%s %s %s %d %d %d %d %d %d %d %d %d %d",
128 row_css_class,
129 spec_name, href,
130 bash_total,
131 dash_total,
132 ash_total,
133 zsh_total,
134 mksh_total,
135 ksh_total,
136 toysh_total,
137 sush_total,
138 brush_total,
139 osh_total)
140
141 # Turn tabs into spaces - awk mutates the row!
142 gsub(/ /, "\t", row)
143 print row
144}
145' "${tsv_files[@]}"
146}
147
148summary-tsv() {
149 local report=$1
150 local spec_subdir=$2
151
152 local manifest=_tmp/spec/SUITE-compat.txt
153
154 # Can't go at the top level because files might not exist!
155 tsv-row \
156 'ROW_CSS_CLASS' 'name' 'name_HREF' "${SH_LABELS[@]}"
157
158 # total row rows goes at the TOP, so it's in <thead> and not sorted.
159 summary-tsv-row $report $spec_subdir _tmp/spec/$spec_subdir/*.result.tsv
160
161 head -n $NUM_SPEC_TASKS $manifest | sort |
162 while read spec_name; do
163 summary-tsv-row $report $spec_subdir $spec_name
164 done
165}
166
167html-summary-header() {
168 local report=$1
169
170 local prefix=../../..
171 spec-html-head $prefix "$report - Shell Compatibility "
172
173 table-sort-begin "width50"
174
175 cat <<EOF
176<p id="home-link">
177 <a href="/">Root</a> |
178 <a href="https://oils.pub/">oils.pub</a>
179</p>
180
181<h1>$report - Shell Compatibility</h1>
182
183<p>Back to <a href="TOP.html">TOP.html</a>.
184</p>
185EOF
186}
187
188html-summary-footer() {
189 local report=$1
190
191 echo "
192<p>Generated by <code>test/spec-compat.sh</code>.
193</p>
194
195<p><a href="$report.tsv">Raw TSV</a>
196</p>
197"
198 table-sort-end "$report" # The table name
199}
200
201write-summary-html() {
202 local report=$1
203 local spec_subdir=$2
204
205 local dir=_tmp/spec/$spec_subdir
206 local out=$dir/$report.html
207
208 summary-tsv $report $spec_subdir >$dir/$report.tsv
209
210 # The underscores are stripped when we don't want them to be!
211 # Note: we could also put "pretty_heading" in the schema
212
213 here-schema-tsv >$dir/$report.schema.tsv <<EOF
214column_name type
215ROW_CSS_CLASS string
216name string
217name_HREF string
218bash integer
219dash integer
220ash integer
221zsh integer
222mksh integer
223ksh integer
224toysh integer
225sush integer
226brush integer
227osh integer
228EOF
229
230 { html-summary-header "$report"
231 # total row isn't sorted
232 tsv2html --thead-offset 1 $dir/$report.tsv
233 html-summary-footer "$report"
234 } > $out
235
236 log "Comparison: file://$REPO_ROOT/$out"
237}
238
239top-html() {
240 local base_url='../../../web'
241 html-head --title 'Shell Compatibility Reports' \
242 "$base_url/base.css"
243
244 echo '
245 <body class="width35">
246 <style>
247 code { color: green; }
248 </style>
249
250 <p id="home-link">
251 <a href="/">Root</a> |
252 <a href="https://oils.pub/">oils.pub</a>
253 </p>
254 '
255
256 cmark <<'EOF'
257## Shell Compatibility Reports
258
259These reports are based on [spec tests written for Oils][spec-tests].
260
261Here are some summary tables. **Click** on the column headers to sort:
262
263- [Total Passing](PASSING.html)
264 - Each shell gets 1 point for each case we marked passing.
265 - Our assertions are usually based on a **survey** of `bash`, `dash`, `mksh`,
266 `zsh`, and other shells. Assertions from 2016-2018 may favor OSH, but
267 there shouldn't be many of them.
268- [Delta bash](DELTA-bash.html)
269 - Compare each shell's passing count vs. bash
270- [Delta OSH](DELTA-osh.html)
271 - Compare each shell's passing count vs. OSH
272
273### Notes and Caveats
274
275- Some tests may fail for innocuous reasons, e.g. printing `'$'` versus `\$`
276 - Shell authors are welcome to use our test suite, and add assertions.
277- OSH has some features that bash doesn't have.
278 - e.g. I removed `spec/strict-options` so shells aren't penalized for not
279 having these features.
280 - But I left `spec/errexit-osh` in because I think new shells should provide
281 alternatives to the **bugs** in POSIX:
282 [YSH Fixes Shell's Error Handling (`errexit`)](https://oils.pub/release/latest/doc/error-handling.html)
283 - I also think shells should adopt [Simple Word Evaluation in Unix Shell](https://oils.pub/release/latest/doc/simple-word-eval.html) (i.e. deprecate `$IFS`, more so than `zsh`)
284- Other ideas
285 - We could add a "majority agreement" metric, for a more neutral report.
286 - We could add the Smoosh test suite. Results are published on our [quality
287page](https://oils.pub/release/latest/quality.html).
288
289### Shells Compared
290
291- GNU `bash`
292 - <https://www.gnu.org/software/bash/>
293 - running fixed version built for Oils
294- `mksh` - shell on Android, derivative of `pdksh`
295 - <http://www.mirbsd.org/mksh.htm>
296 - running fixed version built for Oils
297- AT&T `ksh`
298 - <https://github.com/ksh93/ksh>
299 - running distro package
300- `toysh`
301 - <https://landley.net/toybox/>
302 - running tarball release
303- `sush`
304 - <https://github.com/shellgei/rusty_bash>
305 - running git HEAD
306- `brush`
307 - <https://github.com/reubeno/brush>
308 - running git HEAD
309- `osh`
310 - <https://github.com/oils-for-unix/oils>
311 - running git HEAD
312
313TODO: Add other shells, and be more specific about versions.
314
315### More Comparisons
316
317Possibly TODO
318
319- Binary size
320- Build times
321- Lines of code?
322 - [Oils has a "compressed" implementation](https://www.oilshell.org/blog/2024/09/line-counts.html)
323- Memory safety
324- Oils has many other:
325 - test suites - `test/gold`, `test/wild`, ...
326 - benchmarks - parse time, runtime, ...
327
328### Links
329
330- Wiki:
331 - [Spec Tests][spec-tests]
332 - [Contributing](https://github.com/oils-for-unix/oils/wiki/Contributing)
333- Zulip: <https://oilshell.zulipchat.com/>
334 - Feel free to send feedback, and ask questions!
335
336[spec-tests]: https://github.com/oils-for-unix/oils/wiki/Spec-Tests
337
338### Features Not Yet Implemented in OSH
339
340We know about these gaps:
341
342- `kill` builtin, `let` keyword - we do have some spec tests for them
343- `coproc` keyword
344
345EOF
346
347 echo '
348 </body>
349</html>
350'
351
352# Notes on big files:
353# - spec/strict-options, errexit-osh - could be in the YSH suite
354# - but then that messes up our historical metrics
355# - or we create a new 'spec-compat' suite?
356# - spec/globignore - a big one for OSH
357
358}
359
360write-compare-html() {
361 local spec_subdir='compat'
362 local dir=_tmp/spec/$spec_subdir
363
364 local out=$dir/TOP.html
365 top-html >$out
366 log "Top-level index: file://$REPO_ROOT/$out"
367
368 if test -n "${QUICKLY:-}"; then
369 return
370 fi
371
372 write-summary-html PASSING "$@"
373 write-summary-html DELTA-osh "$@"
374 write-summary-html DELTA-bash "$@"
375}
376
377# TODO: Publish this script
378multi() { ~/git/tree-tools/bin/multi "$@"; }
379
380deploy() {
381 local epoch=${1:-2025-06-19}
382
383 local dest=$PWD/../pages/spec-compat/$epoch
384
385 local web_dir=$dest/web
386 #rm -r -f $web_dir
387
388 #mkdir -p $web_dir
389
390 find web/ -name '*.js' -o -name '*.css' | multi cp $dest
391
392 pushd _tmp
393 find spec/compat -name '*.html' -o -name '*.tsv' | multi cp $dest/renamed-tmp
394 popd
395
396 # Work around Jekyll rule for Github pages
397 #mv -v $dest/_tmp $dest/renamed-tmp
398
399 tree $dest/
400}
401
402task-five "$@"