OILS / test / spec-cpp.sh View on Github | oils.pub

370 lines, 157 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source test/tsv-lib.sh
22source web/table/html.sh
23
24shopt -s failglob # to debug TSV expansion failure below
25
26OSH_PY=$REPO_ROOT/bin/osh
27YSH_PY=$REPO_ROOT/bin/ysh
28
29# Run with ASAN binary by default. Release overrides this
30OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
31YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
32
33# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
34# sh_spec.py.
35export OILS_GC_ON_EXIT=1
36
37#
38# For translation
39#
40
41run-file() {
42 local spec_name=$1
43 shift
44
45 local spec_file=spec/$spec_name.test.sh
46
47 local suite
48 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
49
50 local spec_subdir
51 case $suite in
52 osh) spec_subdir='osh-cpp' ;;
53 ysh) spec_subdir='ysh-cpp' ;;
54 disabled) spec_subdir='disabled-cpp' ;;
55 *) die "Invalid suite $suite" ;;
56 esac
57
58 local base_dir=_tmp/spec/$spec_subdir
59 mkdir -v -p $base_dir
60
61 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
62
63 local variant='cxx-asan'
64 # TODO: turning on gcalways will find more bugs
65 #local variant='cxx-asan+gcalways'
66
67 sh-spec $spec_file \
68 --timeout 10 \
69 --oils-bin-dir $PWD/bin \
70 --oils-cpp-bin-dir $REPO_ROOT/_bin/$variant \
71 --tsv-output $base_dir/${spec_name}.result.tsv \
72 "$@"
73}
74
75osh-all() {
76 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
77
78 # For debugging hangs
79 #export MAX_PROCS=1
80
81 ninja _bin/cxx-asan/{osh,ysh}
82
83 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
84
85 local spec_subdir=osh-cpp
86
87 local status
88 set +o errexit
89 # $suite $compare_mode
90 test/spec-runner.sh all-parallel \
91 osh compare-cpp $spec_subdir "$@"
92 status=$?
93 set -o errexit
94
95 # Write comparison even if we failed
96 write-compare-html $spec_subdir
97
98 return $status
99}
100
101ysh-all() {
102 ninja _bin/cxx-asan/{osh,ysh}
103
104 local spec_subdir=ysh-cpp
105
106 # $suite $compare_mode
107 test/spec-runner.sh all-parallel \
108 ysh compare-cpp $spec_subdir "$@"
109
110 write-compare-html $spec_subdir
111}
112
113console-row() {
114 ### Print out a histogram of results
115
116 awk '
117FNR == 1 {
118 #print FILENAME > "/dev/stderr"
119}
120FNR != 1 {
121 case_num = $1
122 sh = $2
123 result = $3
124
125 if (sh == "osh") {
126 osh[result] += 1
127 } else if (sh == "osh_cpp") { # bin/osh_cpp
128 oe_py[result] += 1
129 } else if (sh == "osh_ALT") { # _bin/*/osh
130 oe_cpp[result] += 1
131 }
132}
133
134function print_hist(sh, hist) {
135 printf("%s\t", sh)
136
137 k = "pass"
138 printf("%s %4d\t", k, hist[k])
139 k = "FAIL"
140 printf("%s %4d\t", k, hist[k])
141
142 print ""
143
144 # This prints N-I, ok, bug, etc.
145 #for (k in hist) {
146 # printf("%s %s\t", k, hist[k])
147 #}
148
149}
150
151END {
152 print_hist("osh", osh)
153 print_hist("osh_cpp", oe_py)
154 print_hist("osh_ALT", oe_cpp)
155}
156 ' "$@"
157}
158
159console-summary() {
160 ### Report on our progress translating
161
162 local spec_subdir=$1
163
164 # Can't go at the top level because files won't exist!
165 readonly TSV=(_tmp/spec/$spec_subdir/*.result.tsv)
166
167 wc -l "${TSV[@]}"
168
169 for file in "${TSV[@]}"; do
170 echo
171 echo "$file"
172 console-row $file
173 done
174
175 echo
176 echo "TOTAL"
177 console-row "${TSV[@]}"
178}
179
180#
181# HTML
182#
183
184summary-tsv-row() {
185 ### Print one row or the last total row
186
187 local spec_subdir=$1
188 shift
189
190 if test $# -eq 1; then
191 local spec_name=$1
192 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.result.tsv )
193 else
194 local spec_name='TOTAL'
195 local -a tsv_files=( "$@" )
196 fi
197
198 awk -v spec_name=$spec_name '
199# skip the first row
200FNR != 1 {
201 case_num = $1
202 sh = $2
203 result = $3
204
205 if (sh == "osh" || sh == "ysh") {
206 osh[result] += 1
207 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
208 osh_native[result] += 1
209 }
210}
211
212END {
213 num_py = osh["pass"]
214 num_cpp = osh_native["pass"]
215 if (spec_name == "TOTAL") {
216 href = ""
217 } else {
218 href = sprintf("%s.html", spec_name)
219 }
220
221 if (num_py == num_cpp) {
222 row_css_class = "cpp-good" # green
223 }
224
225 row = sprintf("%s %s %s %d %d %d",
226 row_css_class,
227 spec_name, href,
228 num_py,
229 num_cpp,
230 num_py - num_cpp)
231
232 # Turn tabs into spaces - awk mutates the row!
233 gsub(/ /, "\t", row)
234 print row
235}
236' "${tsv_files[@]}"
237}
238
239summary-tsv() {
240 local spec_subdir=$1
241
242 local sh_label
243 local manifest
244
245 case $spec_subdir in
246 osh-cpp)
247 sh_label=osh
248 manifest=_tmp/spec/SUITE-osh.txt
249 ;;
250 ysh-cpp)
251 sh_label=ysh
252 manifest=_tmp/spec/SUITE-ysh.txt
253 ;;
254 *)
255 die "Invalid dir $spec_subdir"
256 ;;
257 esac
258
259 # Can't go at the top level because files might not exist!
260 #echo "ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta"
261 tsv-row \
262 'ROW_CSS_CLASS' 'name' 'name_HREF' ${sh_label}_py ${sh_label}_cpp 'delta'
263
264 # total row rows goes at the TOP, so it's in <thead> and not sorted.
265 summary-tsv-row $spec_subdir _tmp/spec/$spec_subdir/*.result.tsv
266
267 head -n $NUM_SPEC_TASKS $manifest | sort |
268 while read spec_name; do
269 summary-tsv-row $spec_subdir $spec_name
270 done
271}
272
273html-summary-header() {
274 local prefix=../../..
275
276 spec-html-head $prefix 'Passing Spec Tests in C++'
277 table-sort-begin "width50"
278
279 echo '
280<p id="home-link">
281 <!-- The release index is two dirs up -->
282 <a href="../..">Up</a> |
283 <a href="/">oils.pub</a>
284</p>
285
286<h1>Python vs C++</h1>
287
288<p>Here is the total number of passing tests. TODO: we should also verify
289tests that do not pass.
290</p>
291
292<p>Another view: <a href="index.html">index.html</a>.
293</p>
294'
295}
296
297html-summary-footer() {
298 echo '
299<p>Generated by <code>test/spec-cpp.sh</code>.
300</p>
301
302<p><a href="SUMMARY.tsv">Raw TSV</a>
303</p>
304'
305 table-sort-end 'SUMMARY' # The table name
306}
307
308write-compare-html() {
309 local spec_subdir=$1
310
311 local sh_label
312 case $spec_subdir in
313 osh-cpp)
314 sh_label=osh
315 ;;
316 ysh-cpp)
317 sh_label=ysh
318 ;;
319 *)
320 die "Invalid dir $spec_subdir"
321 ;;
322 esac
323
324 local dir=_tmp/spec/$spec_subdir
325 local out=$dir/compare.html
326
327 summary-tsv $spec_subdir >$dir/SUMMARY.tsv
328
329 # The underscores are stripped when we don't want them to be!
330 # Note: we could also put "pretty_heading" in the schema
331
332 here-schema-tsv >$dir/SUMMARY.schema.tsv <<EOF
333column_name type
334ROW_CSS_CLASS string
335name string
336name_HREF string
337${sh_label}_py integer
338${sh_label}_cpp integer
339delta integer
340EOF
341
342 { html-summary-header
343 # total row isn't sorted
344 tsv2html --thead-offset 1 $dir/SUMMARY.tsv
345 html-summary-footer
346 } > $out
347
348 log "Comparison: file://$REPO_ROOT/$out"
349}
350
351#
352# Misc
353#
354
355tsv-demo() {
356 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
357 cat _tmp/arith.tsv
358}
359
360repro() {
361 test/spec.sh alias -r 0 -p > _tmp/a
362 ninja _bin/clang-dbg/osh
363 _bin/clang-dbg/osh _tmp/a
364}
365
366repro-all() {
367 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
368}
369
370task-five "$@"