OILS / benchmarks / gc.sh View on Github | oilshell.org

715 lines, 403 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# benchmarks/gc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
11
12source benchmarks/common.sh # benchmark-html-head
13source benchmarks/cachegrind.sh # with-cachegrind
14source build/dev-shell.sh # R_LIBS_USER
15source test/tsv-lib.sh
16
17readonly BASE_DIR=_tmp/gc
18
19# duplicated in benchmarks/gc-cachegrind.sh
20readonly BASE_DIR_CACHEGRIND=_tmp/gc-cachegrind
21
22# See benchmarks/gperftools.sh. I think the Ubuntu package is very old
23
24download-tcmalloc() {
25 # TODO: move this to ../oil_DEPS ?
26 wget --directory _deps \
27 https://github.com/gperftools/gperftools/releases/download/gperftools-2.10/gperftools-2.10.tar.gz
28
29 # Then ./configure; make; sudo make install
30 # installs in /usr/local/lib
31
32 # Note: there's a warning about libunwind -- maybe install that first. Does
33 # it only apply to CPU profiles?
34}
35
36debug-tcmalloc() {
37 touch mycpp/marksweep_heap.cc
38
39 # No evidence of difference
40 for bin in _bin/cxx-{opt,opt+tcmalloc}/osh; do
41 echo $bin
42 ninja $bin
43
44 ldd $bin
45 echo
46
47 ls -l $bin
48 echo
49
50 # Check what we're linking against
51 nm $bin | egrep -i 'malloc|calloc'
52 #wc -l
53 echo
54 done
55}
56
57install-m32() {
58 # needed to compile with -m32
59 sudo apt-get install gcc-multilib g++-multilib
60}
61
62max-rss() {
63 # %e is real time
64 /usr/bin/time --format '%e %M' -- "$@"
65}
66
67compare-m32() {
68 for bin in _bin/cxx-opt{,32}/osh; do
69 echo $bin
70 ninja $bin
71
72 ldd $bin
73 echo
74
75 file $bin
76 echo
77
78 ls -l $bin
79 echo
80
81 # 141136 KiB vs. 110924 KiB. Significant savings, but it's slower.
82 max-rss $bin --ast-format none -n benchmarks/testdata/configure-coreutils
83
84 done
85}
86
87banner() {
88 echo -----
89 echo "$@"
90}
91
92print-tasks() {
93 local -a workloads=(
94 parse.configure-coreutils
95 parse.configure-cpython
96 parse.abuild
97 ex.bashcomp-parse-help # only runs with bash
98 ex.abuild-print-help # bash / dash / zsh
99 ex.compute-fib # bash / dash / zsh
100 )
101
102 local -a shells=(
103 "bash$TAB-"
104 "dash$TAB-"
105 "zsh$TAB-"
106
107 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
108 "_bin/cxx-opt+bumproot/osh${TAB}mut"
109
110 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
111 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
112 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
113
114 # these have trivial GC stats
115 "_bin/cxx-opt/osh${TAB}mut+alloc"
116 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
117 # good GC stats
118 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
119 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc+exit"
120 )
121
122 if test -n "${TCMALLOC:-}"; then
123 shells+=(
124 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc"
125 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free"
126 "_bin/cxx-opt+tcmalloc/osh${TAB}mut+alloc+free+gc"
127 )
128 fi
129
130 local id=0
131
132 for workload in "${workloads[@]}"; do
133 for shell in "${shells[@]}"; do
134 local row_part="$workload${TAB}$shell"
135
136 # Skip these rows
137 case $row_part in
138 "ex.bashcomp-parse-help${TAB}dash"*)
139 continue
140 ;;
141 "ex.bashcomp-parse-help${TAB}zsh"*)
142 continue
143 ;;
144 esac
145
146 local join_id="gc-$id"
147 local row="$join_id${TAB}$row_part"
148 echo "$row"
149
150 id=$((id + 1))
151
152 done
153
154 # Run a quick 10 tasks
155 if test -n "${QUICKLY:-}" && test $id -gt 10; then
156 break
157 fi
158 done
159}
160
161print-cachegrind-tasks() {
162 local -a workloads=(
163 # coreutils is on osh-parser
164 #parse.configure-coreutils
165
166 #parse.configure-cpython
167
168 # Faster tasks, like benchmarks/uftrace, which is instrumented
169 parse.abuild
170 ex.compute-fib
171 )
172
173 local -a shells=(
174 "bash${TAB}-"
175 "_bin/cxx-opt+bumpleak/osh${TAB}mut"
176 "_bin/cxx-opt+bumproot/osh${TAB}mut"
177
178 "_bin/cxx-opt+bumpsmall/osh${TAB}mut+alloc"
179 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc"
180 "_bin/cxx-opt+nopool/osh${TAB}mut+alloc+free+gc"
181
182 "_bin/cxx-opt/osh${TAB}mut+alloc"
183 "_bin/cxx-opt/osh${TAB}mut+alloc+free"
184 "_bin/cxx-opt/osh${TAB}mut+alloc+free+gc"
185 )
186
187 local id=0
188 for workload in "${workloads[@]}"; do
189 for shell in "${shells[@]}"; do
190 local row_part="$workload${TAB}$shell"
191
192 local join_id="cachegrind-$id"
193 local row="$join_id${TAB}$row_part"
194 echo "$row"
195
196 id=$((id + 1))
197 done
198 done
199 #print-tasks | egrep 'configure-coreutils' | egrep osh
200}
201
202
203readonly BIG_THRESHOLD=$(( 1 * 1000 * 1000 * 1000 )) # 1 B
204
205run-tasks() {
206 local tsv_out=$1
207 local mode=${2:-time}
208
209 while read -r join_id task sh_path shell_runtime_opts; do
210
211 # Parse different files
212 case $task in
213 parse.configure-coreutils)
214 data_file='benchmarks/testdata/configure-coreutils'
215 ;;
216 parse.configure-cpython)
217 data_file='Python-2.7.13/configure'
218 ;;
219 parse.abuild)
220 data_file='benchmarks/testdata/abuild'
221 ;;
222 esac
223
224 # Construct argv for each task
225 local -a argv
226 case $task in
227 parse.*)
228 argv=( -n $data_file )
229
230 case $sh_path in
231 _bin/*/osh)
232 argv=( --ast-format none "${argv[@]}" )
233 ;;
234 esac
235 ;;
236
237 ex.bashcomp-parse-help)
238 argv=( benchmarks/parse-help/pure-excerpt.sh parse_help_file
239 benchmarks/parse-help/clang.txt )
240 ;;
241
242 ex.abuild-print-help)
243 argv=( testdata/osh-runtime/abuild -h )
244 ;;
245
246 ex.compute-fib)
247 # fewer iterations when instrumented
248 local iters
249 if test $mode = time; then
250 iters=100
251 else
252 iters=10
253 fi
254
255 argv=( benchmarks/compute/fib.sh $iters 44 )
256 ;;
257
258 *)
259 die "Invalid task $task"
260 ;;
261 esac
262
263 echo $join_id $task $sh_path $shell_runtime_opts
264
265 argv=( $sh_path "${argv[@]}" )
266 #echo + "${argv[@]}"
267 #set -x
268
269 if test $mode = cachegrind; then
270 # Add prefix
271 argv=( $0 with-cachegrind $BASE_DIR_CACHEGRIND/raw/$join_id.txt "${argv[@]}" )
272 fi
273
274 # Wrap in a command that writes one row of a TSV
275 # Note: for cachegrind, we need the join ID, but the --rusage is meaningless
276 local -a instrumented=(
277 time-tsv -o $tsv_out --append
278 --rusage
279 --field "$join_id" --field "$task" --field "$sh_path"
280 --field "$shell_runtime_opts"
281 -- "${argv[@]}"
282 )
283
284 # Run with the right environment variables
285
286 case $shell_runtime_opts in
287 -)
288 "${instrumented[@]}" > /dev/null
289 ;;
290 mut)
291 OILS_GC_STATS=1 \
292 "${instrumented[@]}" > /dev/null
293 ;;
294 mut+alloc)
295 # disable GC with big threshold
296 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD \
297 "${instrumented[@]}" > /dev/null
298 ;;
299 mut+alloc+free)
300 # do a single GC on exit
301 OILS_GC_STATS=1 OILS_GC_THRESHOLD=$BIG_THRESHOLD OILS_GC_ON_EXIT=1 \
302 "${instrumented[@]}" > /dev/null
303 ;;
304 mut+alloc+free+gc)
305 # Default configuration
306 #
307 # Save the GC stats here. None of the other runtime options are that
308 # interesting.
309
310 if test $mode = 'time' && test $sh_path != _bin/cxx-opt+nopool/osh; then
311 OILS_GC_STATS_FD=99 \
312 "${instrumented[@]}" > /dev/null 99>$BASE_DIR/raw/$join_id.txt
313 else
314 "${instrumented[@]}" > /dev/null
315 fi
316 ;;
317 mut+alloc+free+gc+exit)
318 # also GC on exit
319 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
320 "${instrumented[@]}" > /dev/null
321 ;;
322
323 *)
324 die "Invalid shell runtime opts $shell_runtime_opts"
325 ;;
326 esac
327
328 done
329
330 # TODO: OILS_GC_STATS_FD and tsv_column_from_files.py
331}
332
333fd-demo() {
334 local out=_tmp/gc/demo.txt
335
336 local bin=_bin/cxx-dbg/oils-for-unix
337 ninja $bin
338
339 # Hm you can't do $fd>out.txt, but that's OK
340 local fd=99
341
342 OILS_GC_STATS_FD=$fd 99>$out \
343 $bin --ast-format none -n benchmarks/testdata/configure
344
345 ls -l $out
346 cat $out
347}
348
349more-variants() {
350 # TODO: could revive this
351
352 case $compare_more in
353 (*m32*)
354 # Surprisingly, -m32 is SLOWER, even though it allocates less.
355 # My guess is because less work is going into maintaining this code path in
356 # GCC.
357
358 # 223 ms
359 # 61.9 MB bytes allocated
360 local bin=_bin/cxx-opt32/oils-for-unix
361 OILS_GC_THRESHOLD=$big_threshold \
362 run-osh $tsv_out $bin 'm32 mutator+malloc' $file
363
364 # 280 ms
365 OILS_GC_STATS=1 \
366 run-osh $tsv_out $bin 'm32 mutator+malloc+free+gc' $file
367 ;;
368 esac
369
370 # Show log of GC
371 case $compare_more in
372 (*gcverbose*)
373 local bin=_bin/cxx-gcverbose/oils-for-unix
374 # 280 ms
375 OILS_GC_STATS=1 OILS_GC_ON_EXIT=1 \
376 run-osh $tsv_out $bin 'gcverbose mutator+malloc+free+gc' $file
377 ;;
378 esac
379
380 if command -v pretty-tsv; then
381 pretty-tsv $tsv_out
382 fi
383}
384
385build-binaries() {
386 local -a bin=( _bin/cxx-opt{,+bumpleak,+bumproot,+bumpsmall,+nopool}/osh )
387
388 if test -n "${TCMALLOC:-}"; then
389 bin+=( _bin/cxx-opt+tcmalloc/osh )
390 fi
391 ninja "${bin[@]}"
392}
393
394measure-all() {
395 build-binaries
396
397 local tsv_out=${1:-$BASE_DIR/raw/times.tsv}
398 mkdir -p $(dirname $tsv_out)
399
400 # Make the header
401 time-tsv -o $tsv_out --print-header \
402 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
403
404 time print-tasks | run-tasks $tsv_out
405
406 if command -v pretty-tsv; then
407 pretty-tsv $tsv_out
408 fi
409}
410
411measure-cachegrind() {
412 build-binaries
413
414 local tsv_out=${1:-$BASE_DIR_CACHEGRIND/raw/times.tsv}
415
416 mkdir -p $(dirname $tsv_out)
417
418 # Make the header
419 time-tsv -o $tsv_out --print-header \
420 --rusage --field join_id --field task --field sh_path --field shell_runtime_opts
421
422 print-cachegrind-tasks | run-tasks $tsv_out cachegrind
423
424 # TODO: join cachegrind columns
425
426 if command -v pretty-tsv; then
427 pretty-tsv $tsv_out
428 fi
429}
430
431print-report() {
432 local in_dir=$1
433
434 benchmark-html-head 'Memory Management Overhead'
435
436 cat <<EOF
437 <body class="width60">
438 <p id="home-link">
439 <a href="/">oilshell.org</a>
440 </p>
441EOF
442
443 cmark << 'EOF'
444## Memory Management Overhead
445
446Source code: [oil/benchmarks/gc.sh](https://github.com/oilshell/oil/tree/master/benchmarks/gc.sh)
447EOF
448
449 cmark << 'EOF'
450### GC Stats
451
452EOF
453
454 tsv2html $in_dir/gc_stats.tsv
455
456 cmark << 'EOF'
457
458- Underlying data: [stage2/gc_stats.tsv](stage2/gc_stats.tsv)
459- More columns: [stage1/gc_stats.tsv](stage1/gc_stats.tsv)
460
461### Resource Usage
462
463#### parse.configure-cpython
464
465EOF
466
467 tsv2html $in_dir/parse.configure-cpython.tsv
468
469 cmark << 'EOF'
470#### parse.configure-coreutils
471
472Parsing the autoconf-generated `configure` script from GNU coreutils.
473
474Note that unlike other shells, `osh -n` retains all nodes on purpose. (See the
475[parser benchmark](../osh-parser/index.html)).
476
477EOF
478
479 tsv2html $in_dir/parse.configure-coreutils.tsv
480
481 cmark <<'EOF'
482#### parse.abuild
483
484Parsing `abuild` from Alpine Linux.
485EOF
486
487 tsv2html $in_dir/parse.abuild.tsv
488
489 cmark <<'EOF'
490#### ex.compute-fib
491
492A synthetic benchmark for POSIX shell arithmetic.
493EOF
494
495 tsv2html $in_dir/ex.compute-fib.tsv
496
497 cmark <<'EOF'
498#### ex.bashcomp-parse-help
499
500A realistic `bash-completion` workload.
501EOF
502
503 tsv2html $in_dir/ex.bashcomp-parse-help.tsv
504
505 cmark <<'EOF'
506#### ex.abuild-print-help
507
508Running `abuild -h` from Alpine Linux.
509
510EOF
511
512 tsv2html $in_dir/ex.abuild-print-help.tsv
513
514 cmark << 'EOF'
515- Underlying data: [stage2/times.tsv](stage2/times.tsv)
516EOF
517
518 cat <<EOF
519
520 </body>
521</html>
522EOF
523}
524
525make-report() {
526 mkdir -p $BASE_DIR/{stage1,stage2}
527
528 # Concatenate tiny files
529 benchmarks/gc_stats_to_tsv.py $BASE_DIR/raw/gc-*.txt \
530 > $BASE_DIR/stage1/gc_stats.tsv
531
532 # Make TSV files
533 benchmarks/report.R gc $BASE_DIR $BASE_DIR/stage2
534
535 # Make HTML
536 benchmarks/report.sh stage3 $BASE_DIR
537}
538
539soil-run() {
540 ### Run in soil/benchmarks
541
542 measure-all
543
544 make-report
545}
546
547#
548# Misc Tests
549#
550
551gc-parse-smoke() {
552 local variant=${1:-opt}
553 local file=${2:-configure}
554
555 local bin=_bin/cxx-$variant/osh
556 ninja $bin
557
558 # OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
559 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
560 $bin --ast-format none -n $file
561
562 # No leaks
563 # OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 $bin -n -c '('
564}
565
566gc-parse-big() {
567 local variant=${1:-opt}
568
569 gc-parse-smoke $variant benchmarks/testdata/configure-coreutils
570}
571
572gc-run-smoke() {
573 local variant=${1:-opt}
574
575 local bin=_bin/cxx-$variant/oils-for-unix
576 ninja $bin
577
578 # expose a bug with printf
579 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=500 OILS_GC_ON_EXIT=1 \
580 $bin -c 'for i in $(seq 100); do printf "%s\\n" "-- $i"; done'
581}
582
583gc-run-oil() {
584 ### Run some scripts from the repo
585
586 local variant=${1:-opt}
587
588 local bin=_bin/cxx-$variant/oils-for-unix
589 ninja $bin
590
591 local i=0
592 for script in */*.sh; do
593 case $script in
594 (build/clean.sh|build/common.sh|build/dev.sh)
595 # Top level does something!
596 echo "=== SKIP $script"
597 continue
598 ;;
599 esac
600
601 echo
602 echo "=== ($i) $script"
603
604 # Just run the top level, which (hopefully) does nothing
605 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=1000 OILS_GC_ON_EXIT=1 \
606 $bin $script
607
608 i=$((i + 1))
609 if test $i -gt 60; then
610 break
611 fi
612 done
613}
614
615gc-run-big() {
616 local variant=${1:-opt}
617
618 local target=_bin/cxx-$variant/oils-for-unix
619 ninja $target
620
621 local osh=$REPO_ROOT/$target
622
623 local dir=_tmp/gc-run-big
624 rm -r -f -v $dir
625 mkdir -v -p $dir
626
627 pushd $dir
628 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 OILS_GC_THRESHOLD=100000 OILS_GC_ON_EXIT=1 \
629 $osh ../../Python-2.7.13/configure
630 popd
631}
632
633run-verbose() {
634 _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
635 /usr/bin/time --format '*** MAX RSS KiB = %M' -- \
636 "$@"
637}
638
639# This hit the 24-bit object ID limitation in 2.5 seconds
640# Should be able to run indefinitely.
641run-for-a-long-time() {
642 local bin=_bin/cxx-opt/osh
643 ninja $bin
644 run-verbose $bin benchmarks/compute/fib.sh 10000
645
646 # time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 _bin/cxx-opt/osh benchmarks/compute/fib.sh 10000
647}
648
649while-loop() {
650 local i=0
651 while test $i -lt 10000; do
652 if ((i % 1000 == 0)) ; then
653 echo $i
654 fi
655 i=$((i + 1))
656 continue # BUG: skipped GC point
657 done
658}
659
660for-loop() {
661 for i in $(seq 10000); do
662 if ((i % 1000 == 0)) ; then
663 echo $i
664 fi
665 continue
666 done
667}
668
669recurse() {
670 local n=${1:-3000}
671
672 if ((n % 100 == 0)) ; then
673 echo $n
674 fi
675
676 if test $n = 0; then
677 return
678 fi
679
680 recurse $((n - 1))
681}
682
683test-loops() {
684 ### Regression for leak
685
686 local bin=_bin/cxx-opt/osh
687 ninja $bin
688
689 run-verbose $bin $0 recurse
690 echo
691
692 run-verbose $bin $0 while-loop
693 echo
694
695 run-verbose $bin $0 for-loop
696}
697
698expand-loop() {
699 local n=$1
700
701 local bin=_bin/cxx-opt/osh
702 ninja $bin
703
704 set -x
705 time _OILS_GC_VERBOSE=1 OILS_GC_STATS=1 \
706 $bin -c "for i in {1..$n}; do echo \$i; done > /dev/null"
707 set +x
708}
709
710test-brace-exp() {
711 expand-loop 330000
712 expand-loop 340000
713}
714
715"$@"