OILS / benchmarks / gc.sh View on Github | oils.pub

739 lines, 413 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 # TODO: move this to ../oil_DEPS ?
26 wget --directory _deps \
27 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
28
29 # Then ./configure; make; sudo make install
30 # installs in /usr/local/lib
31
32 # Note: there's a warning about libunwind -- maybe install that first. Does
33 # it only apply to CPU profiles?
34}
35
36debug-tcmalloc() {
37 touch mycpp/marksweep_heap.cc
38
39 # No evidence of difference
40 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
41 echo $bin
42 ninja $bin
43
44 ldd $bin
45 echo
46
47 ls -l $bin
48 echo
49
50 # Check what we're linking against
51 nm $bin | egrep -i 'malloc|calloc'
52 #wc -l
53 echo
54 done
55}
56
57install-m32() {
58 # needed to compile with -m32
59 sudo apt-get install gcc-multilib g++-multilib
60}
61
62max-rss() {
63 # %e is real time
64 /usr/bin/time --format '%e %M' -- "$@"
65}
66
67compare-m32() {
68 for bin in _bin/cxx-opt{,32}/osh; do
69 echo $bin
70 ninja $bin
71
72 ldd $bin
73 echo
74
75 file $bin
76 echo
77
78 ls -l $bin
79 echo
80
81 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
82 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
83
84 done
85}
86
87print-tasks() {
88 local mycpp_souffle=${1:-}
89
90 local -a workloads=(
91 parse.configure-coreutils
92 parse.configure-cpython
93 parse.abuild
94 ex.bashcomp-parse-help # only runs with bash
95 ex.abuild-print-help # bash / dash / zsh
96 ex.compute-fib # bash / dash / zsh
97 )
98
99 local -a shells=(
100 "bash$TAB-"
101 "dash$TAB-"
102 "zsh$TAB-"
103
104 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
105 "_bin/cxx-opt+bumproot/osh${TAB}mut"
106
107 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
108 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
109 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
110
111 # these have trivial GC stats
112 "_bin/cxx-opt/osh${TAB}mut+alloc"
113 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
114 # good GC stats
115 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
116 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
117 )
118
119 if test -n "$mycpp_souffle"; then
120 shells+=(
121 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc"
122 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free"
123 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
124 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc+exit"
125 )
126 fi
127
128 if test -n "${TCMALLOC:-}"; then
129 shells+=(
130 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
131 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
132 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
133 )
134 fi
135
136 local id=0
137
138 for workload in "${workloads[@]}"; do
139 for shell in "${shells[@]}"; do
140 local row_part="$workload${TAB}$shell"
141
142 # Skip these rows
143 case $row_part in
144 "ex.bashcomp-parse-help${TAB}dash"*)
145 continue
146 ;;
147 "ex.bashcomp-parse-help${TAB}zsh"*)
148 continue
149 ;;
150 esac
151
152 local join_id="gc-$id"
153 local row="$join_id${TAB}$row_part"
154 echo "$row"
155
156 id=$((id + 1))
157
158 done
159
160 # Run a quick 10 tasks
161 if test -n "${QUICKLY:-}" && test $id -gt 10; then
162 break
163 fi
164 done
165}
166
167print-cachegrind-tasks() {
168 local mycpp_souffle=${1:-}
169
170 local -a workloads=(
171 # coreutils is on osh-parser
172 #parse.configure-coreutils
173
174 #parse.configure-cpython
175
176 # Faster tasks, like benchmarks/uftrace, which is instrumented
177 parse.abuild
178 ex.compute-fib
179 )
180
181 local -a shells=(
182 "bash${TAB}-"
183 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
184 "_bin/cxx-opt+bumproot/osh${TAB}mut"
185
186 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
187
188 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
189
190 "_bin/cxx-opt/osh${TAB}mut+alloc"
191 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
192 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
193 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
194 )
195
196 if test -n "$mycpp_souffle"; then
197 shells+=(
198 "_bin/cxx-opt/mycpp-souffle/osh${TAB}mut+alloc+free+gc"
199 )
200 fi
201
202 local id=0
203 for workload in "${workloads[@]}"; do
204 for shell in "${shells[@]}"; do
205 local row_part="$workload${TAB}$shell"
206
207 local join_id="cachegrind-$id"
208 local row="$join_id${TAB}$row_part"
209 echo "$row"
210
211 id=$((id + 1))
212 done
213 done
214 #print-tasks | egrep 'configure-coreutils' | egrep osh
215}
216
217
218readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
219
220run-tasks() {
221 local tsv_out=$1
222 local mode=${2:-time}
223
224 while read -r join_id task sh_path shell_runtime_opts; do
225
226 # Parse different files
227 case $task in
228 parse.configure-coreutils)
229 data_file='benchmarks/testdata/configure-coreutils'
230 ;;
231 parse.configure-cpython)
232 data_file='Python-2.7.13/configure'
233 ;;
234 parse.abuild)
235 data_file='benchmarks/testdata/abuild'
236 ;;
237 esac
238
239 # Construct argv for each task
240 local -a argv
241 case $task in
242 parse.*)
243 argv=( -n $data_file )
244
245 case $sh_path in
246 _bin/*/osh)
247 argv=( --ast-format none "${argv[@]}" )
248 ;;
249 esac
250 ;;
251
252 ex.bashcomp-parse-help)
253 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
254 benchmarks/parse-help/clang.txt )
255 ;;
256
257 ex.abuild-print-help)
258 argv=( testdata/osh-runtime/abuild -h )
259 ;;
260
261 ex.compute-fib)
262 # fewer iterations when instrumented
263 local iters
264 if test $mode = time; then
265 iters=100
266 else
267 iters=10
268 fi
269
270 argv=( benchmarks/compute/fib.sh $iters 44 )
271 ;;
272
273 *)
274 die "Invalid task $task"
275 ;;
276 esac
277
278 echo $join_id $task $sh_path $shell_runtime_opts
279
280 argv=( $sh_path "${argv[@]}" )
281 #echo + "${argv[@]}"
282 #set -x
283
284 if test $mode = cachegrind; then
285 # Add prefix
286 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
287 fi
288
289 # Wrap in a command that writes one row of a TSV
290 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
291 local -a instrumented=(
292 time-tsv -o $tsv_out --append
293 --rusage
294 --field "$join_id" --field "$task" --field "$sh_path"
295 --field "$shell_runtime_opts"
296 -- "${argv[@]}"
297 )
298
299 # Run with the right environment variables
300
301 case $shell_runtime_opts in
302 -)
303 "${instrumented[@]}" > /dev/null
304 ;;
305 mut)
306 OILS_GC_STATS=1 \
307 "${instrumented[@]}" > /dev/null
308 ;;
309 mut+alloc)
310 # disable GC with big threshold
311 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
312 "${instrumented[@]}" > /dev/null
313 ;;
314 mut+alloc+free)
315 # do a single GC on exit
316 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
317 "${instrumented[@]}" > /dev/null
318 ;;
319 mut+alloc+free+gc)
320 # Default configuration
321 #
322 # Save the GC stats here. None of the other runtime options are that
323 # interesting.
324
325 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
326 OILS_GC_STATS_FD=99 \
327 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
328 else
329 "${instrumented[@]}" > /dev/null
330 fi
331 ;;
332 mut+alloc+free+gc+exit)
333 # also GC on exit
334 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
335 "${instrumented[@]}" > /dev/null
336 ;;
337
338 *)
339 die "Invalid shell runtime opts $shell_runtime_opts"
340 ;;
341 esac
342
343 done
344
345 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
346}
347
348fd-demo() {
349 local out=_tmp/gc/demo.txt
350
351 local bin=_bin/cxx-dbg/oils-for-unix
352 ninja $bin
353
354 # Hm you can't do $fd>out.txt, but that's OK
355 local fd=99
356
357 OILS_GC_STATS_FD=$fd 99>$out \
358 $bin --ast-format none -n benchmarks/testdata/configure
359
360 ls -l $out
361 cat $out
362}
363
364more-variants() {
365 # TODO: could revive this
366
367 case $compare_more in
368 (*m32*)
369 # Surprisingly, -m32 is SLOWER, even though it allocates less.
370 # My guess is because less work is going into maintaining this code path in
371 # GCC.
372
373 # 223 ms
374 # 61.9 MB bytes allocated
375 local bin=_bin/cxx-opt32/oils-for-unix
376 OILS_GC_THRESHOLD=$big_threshold \
377 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
378
379 # 280 ms
380 OILS_GC_STATS=1 \
381 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
382 ;;
383 esac
384
385 # Show log of GC
386 case $compare_more in
387 (*gcverbose*)
388 local bin=_bin/cxx-gcverbose/oils-for-unix
389 # 280 ms
390 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
391 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
392 ;;
393 esac
394
395 if command -v pretty-tsv; then
396 pretty-tsv $tsv_out
397 fi
398}
399
400build-binaries() {
401 soil/cpp-tarball.sh build-like-ninja \
402 opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}
403
404 OILS_TRANSLATOR=mycpp-souffle soil/cpp-tarball.sh build-like-ninja opt
405}
406
407measure-all() {
408 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
409 local mycpp_souffle=${2:-}
410
411 build-binaries
412
413 mkdir -p $(dirname $tsv_out)
414
415 # Make the header
416 time-tsv -o $tsv_out --print-header \
417 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
418
419 # Pass through args, which may include mycpp-souffle
420 time print-tasks "$mycpp_souffle" | run-tasks $tsv_out
421
422 if command -v pretty-tsv; then
423 pretty-tsv $tsv_out
424 fi
425}
426
427measure-cachegrind() {
428 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
429 local mycpp_souffle=${2:-T}
430
431 build-binaries
432
433 mkdir -p $(dirname $tsv_out)
434
435 # Make the header
436 time-tsv -o $tsv_out --print-header \
437 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
438
439 print-cachegrind-tasks "$mycpp_souffle" | run-tasks $tsv_out cachegrind
440
441 # TODO: join cachegrind columns
442
443 if command -v pretty-tsv; then
444 pretty-tsv $tsv_out
445 fi
446}
447
448print-report() {
449 local in_dir=$1
450
451 benchmark-html-head 'Memory Management Overhead'
452
453 cat <<EOF
454 <body class="width60">
455 <p id="home-link">
456 <a href="/">oils.pub</a>
457 </p>
458EOF
459
460 cmark << 'EOF'
461## Memory Management Overhead
462
463Source code: [oils/benchmarks/gc.sh](https://github.com/oils-for-unix/oils/tree/master/benchmarks/gc.sh)
464EOF
465
466 cmark << 'EOF'
467### GC Stats
468
469EOF
470
471 tsv2html $in_dir/gc_stats.tsv
472
473 cmark << 'EOF'
474
475- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
476- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
477
478### Resource Usage
479
480#### parse.configure-cpython
481
482EOF
483
484 tsv2html $in_dir/parse.configure-cpython.tsv
485
486 cmark << 'EOF'
487#### parse.configure-coreutils
488
489Parsing the autoconf-generated `configure` script from GNU coreutils.
490
491Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
492[parser benchmark](../osh-parser/index.html)).
493
494EOF
495
496 tsv2html $in_dir/parse.configure-coreutils.tsv
497
498 cmark <<'EOF'
499#### parse.abuild
500
501Parsing `abuild` from Alpine Linux.
502EOF
503
504 tsv2html $in_dir/parse.abuild.tsv
505
506 cmark <<'EOF'
507#### ex.compute-fib
508
509A synthetic benchmark for POSIX shell arithmetic.
510EOF
511
512 tsv2html $in_dir/ex.compute-fib.tsv
513
514 cmark <<'EOF'
515#### ex.bashcomp-parse-help
516
517A realistic `bash-completion` workload.
518EOF
519
520 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
521
522 cmark <<'EOF'
523#### ex.abuild-print-help
524
525Running `abuild -h` from Alpine Linux.
526
527EOF
528
529 tsv2html $in_dir/ex.abuild-print-help.tsv
530
531 cmark << 'EOF'
532- Underlying data: [stage2/times.tsv](stage2/times.tsv)
533EOF
534
535 cat <<EOF
536
537 </body>
538</html>
539EOF
540}
541
542make-report() {
543 mkdir -p $BASE_DIR/{stage1,stage2}
544
545 # Concatenate tiny files
546 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
547 > $BASE_DIR/stage1/gc_stats.tsv
548
549 # Make TSV files
550 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
551
552 # Make HTML
553 benchmarks/report.sh stage3 $BASE_DIR
554}
555
556soil-run() {
557 ### Run in soil/benchmarks
558
559 measure-all '' mycpp-souffle
560
561 make-report
562}
563
564run-for-release() {
565 # TODO: turn on Souffle
566 measure-all ''
567
568 make-report
569}
570
571#
572# Misc Tests
573#
574
575gc-parse-smoke() {
576 local variant=${1:-opt}
577 local file=${2:-configure}
578
579 local bin=_bin/cxx-$variant/osh
580 ninja $bin
581
582 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
583 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
584 $bin --ast-format none -n $file
585
586 # No leaks
587 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
588}
589
590gc-parse-big() {
591 local variant=${1:-opt}
592
593 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
594}
595
596gc-run-smoke() {
597 local variant=${1:-opt}
598
599 local bin=_bin/cxx-$variant/oils-for-unix
600 ninja $bin
601
602 # expose a bug with printf
603 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
604 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
605}
606
607gc-run-oil() {
608 ### Run some scripts from the repo
609
610 local variant=${1:-opt}
611
612 local bin=_bin/cxx-$variant/oils-for-unix
613 ninja $bin
614
615 local i=0
616 for script in */*.sh; do
617 case $script in
618 (build/clean.sh|build/common.sh|build/dev.sh)
619 # Top level does something!
620 echo "=== SKIP $script"
621 continue
622 ;;
623 esac
624
625 echo
626 echo "=== ($i) $script"
627
628 # Just run the top level, which (hopefully) does nothing
629 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
630 $bin $script
631
632 i=$((i + 1))
633 if test $i -gt 60; then
634 break
635 fi
636 done
637}
638
639gc-run-big() {
640 local variant=${1:-opt}
641
642 local target=_bin/cxx-$variant/oils-for-unix
643 ninja $target
644
645 local osh=$REPO_ROOT/$target
646
647 local dir=_tmp/gc-run-big
648 rm -r -f -v $dir
649 mkdir -v -p $dir
650
651 pushd $dir
652 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
653 $osh ../../Python-2.7.13/configure
654 popd
655}
656
657run-verbose() {
658 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
659 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
660 "$@"
661}
662
663# This hit the 24-bit object ID limitation in 2.5 seconds
664# Should be able to run indefinitely.
665run-for-a-long-time() {
666 local bin=_bin/cxx-opt/osh
667 ninja $bin
668 run-verbose $bin benchmarks/compute/fib.sh 10000
669
670 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
671}
672
673while-loop() {
674 local i=0
675 while test $i -lt 10000; do
676 if ((i % 1000 == 0)) ; then
677 echo $i
678 fi
679 i=$((i + 1))
680 continue # BUG: skipped GC point
681 done
682}
683
684for-loop() {
685 for i in $(seq 10000); do
686 if ((i % 1000 == 0)) ; then
687 echo $i
688 fi
689 continue
690 done
691}
692
693recurse() {
694 local n=${1:-3000}
695
696 if ((n % 100 == 0)) ; then
697 echo $n
698 fi
699
700 if test $n = 0; then
701 return
702 fi
703
704 recurse $((n - 1))
705}
706
707test-loops() {
708 ### Regression for leak
709
710 local bin=_bin/cxx-opt/osh
711 ninja $bin
712
713 run-verbose $bin $0 recurse
714 echo
715
716 run-verbose $bin $0 while-loop
717 echo
718
719 run-verbose $bin $0 for-loop
720}
721
722expand-loop() {
723 local n=$1
724
725 local bin=_bin/cxx-opt/osh
726 ninja $bin
727
728 set -x
729 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
730 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
731 set +x
732}
733
734test-brace-exp() {
735 expand-loop 330000
736 expand-loop 340000
737}
738
739"$@"