OILS / benchmarks / osh-runtime.sh View on Github | oilshell.org

565 lines, 327 significant
1#!/usr/bin/env bash
2#
3# Test scripts found in the wild for both correctness and performance.
4#
5# Usage:
6# benchmarks/osh-runtime.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13
14source benchmarks/common.sh # tsv-concat
15source benchmarks/id.sh # print-job-id
16source test/common.sh
17source test/tsv-lib.sh # tsv-row
18
19readonly BASE_DIR=_tmp/osh-runtime
20
21# TODO: Move to ../oil_DEPS
22readonly TAR_DIR=$PWD/_deps/osh-runtime # Make it absolute
23
24#
25# Dependencies
26#
27
28readonly PY27_DIR=$PWD/Python-2.7.13
29
30# NOTE: Same list in oilshell.org/blob/run.sh.
31tarballs() {
32 cat <<EOF
33tcc-0.9.26.tar.bz2
34yash-2.46.tar.xz
35ocaml-4.06.0.tar.xz
36util-linux-2.40.tar.xz
37EOF
38}
39
40download() {
41 mkdir -p $TAR_DIR
42 tarballs | xargs -n 1 -I {} --verbose -- \
43 wget --no-clobber --directory $TAR_DIR 'https://www.oilshell.org/blob/testdata/{}'
44}
45
46extract() {
47 set -x
48 time for f in $TAR_DIR/*.{bz2,xz}; do
49 tar -x --directory $TAR_DIR --file $f
50 done
51 set +x
52
53 ls -l $TAR_DIR
54}
55
56#
57# Computation
58#
59
60run-tasks() {
61 local raw_out_dir=$1
62 raw_out_dir="$PWD/$raw_out_dir" # because we change dirs
63
64 # Bug fix for dynamic scoping!
65 local host_name sh_path workload
66
67 local task_id=0
68 while read -r host_name sh_path workload; do
69
70 log "*** $host_name $sh_path $workload $task_id"
71
72 local sh_run_path
73 case $sh_path in
74 /*) # Already absolute
75 sh_run_path=$sh_path
76 ;;
77 */*) # It's relative, so make it absolute
78 sh_run_path=$PWD/$sh_path
79 ;;
80 *) # 'dash' should remain 'dash'
81 sh_run_path=$sh_path
82 ;;
83 esac
84
85 local working_dir=''
86 local files_out_dir="$raw_out_dir/files-$task_id"
87 mkdir -v -p $files_out_dir
88
89 local save_new_files=''
90
91 local -a argv
92 case $workload in
93 hello-world)
94 argv=( testdata/osh-runtime/hello_world.sh )
95 ;;
96
97 bin-true)
98 argv=( testdata/osh-runtime/bin_true.sh )
99 ;;
100
101 abuild-print-help)
102 argv=( testdata/osh-runtime/abuild -h )
103 ;;
104
105 configure.cpython)
106 argv=( $PY27_DIR/configure )
107 working_dir=$files_out_dir
108 ;;
109
110 configure.util-linux)
111 # flag needed to avoid sqlite3 dep error message
112 argv=( $TAR_DIR/util-linux-2.40/configure --disable-liblastlog2 )
113 working_dir=$files_out_dir
114 ;;
115
116 configure.*)
117 argv=( ./configure )
118
119 local conf_dir
120 case $workload in
121 *.ocaml)
122 conf_dir='ocaml-4.06.0'
123 ;;
124 *.tcc)
125 conf_dir='tcc-0.9.26'
126 ;;
127 *.yash)
128 conf_dir='yash-2.46'
129 ;;
130 *)
131 die "Invalid workload $workload"
132 esac
133
134 # These are run in-tree?
135 working_dir=$TAR_DIR/$conf_dir
136 ;;
137
138 *)
139 die "Invalid workload $workload"
140 ;;
141 esac
142
143 local -a time_argv=(
144 time-tsv
145 --output "$raw_out_dir/times.tsv" --append
146 --rusage
147 --rusage-2
148 --field "$task_id"
149 --field "$host_name" --field "$sh_path"
150 --field "$workload"
151 -- "$sh_run_path" "${argv[@]}"
152 )
153
154 local stdout_file="$files_out_dir/STDOUT.txt"
155 local gc_stats_file="$raw_out_dir/gc-$task_id.txt"
156
157 # Maybe change dirs
158 if test -n "$working_dir"; then
159 pushd "$working_dir"
160 fi
161
162 if test -n "$save_new_files"; then
163 touch __TIMESTAMP
164 fi
165
166 # Run it, possibly with GC stats
167 case $sh_path in
168 *_bin/*/osh)
169 OILS_GC_STATS_FD=99 "${time_argv[@]}" > $stdout_file 99> $gc_stats_file
170 ;;
171 *)
172 "${time_argv[@]}" > $stdout_file
173 ;;
174 esac
175
176 if test -n "$save_new_files"; then
177 echo "COPYING to $files_out_dir"
178 find . -type f -newer __TIMESTAMP \
179 | xargs -I {} -- cp --verbose {} $files_out_dir
180 fi
181
182 # Restore dir
183 if test -n "$working_dir"; then
184 popd
185 fi
186
187 task_id=$((task_id + 1))
188 done
189}
190
191# Sorted by priority for test-oils.sh osh-runtime --num-shells 3
192
193readonly -a ALL_WORKLOADS=(
194 hello-world
195 bin-true
196
197 configure.cpython
198 configure.util-linux
199 configure.ocaml
200 configure.tcc
201 configure.yash
202
203 abuild-print-help
204)
205
206print-workloads() {
207 ### for help
208
209 for w in "${ALL_WORKLOADS[@]}"; do
210 echo " $w"
211 done
212}
213
214print-tasks() {
215 local host_name=$1
216 local osh_native=$2
217
218 if test -n "${QUICKLY:-}"; then
219 workloads=(
220 hello-world
221 bin-true
222 #configure.util-linux
223 #abuild-print-help
224 )
225 else
226 workloads=( "${ALL_WORKLOADS[@]}" )
227 fi
228
229 for sh_path in bash dash bin/osh $osh_native; do
230 for workload in "${workloads[@]}"; do
231 tsv-row $host_name $sh_path $workload
232 done
233 done
234}
235
236print-tasks-xshar() {
237 local host_name=$1
238 local osh_native=$2
239
240 local num_iters=${3:-1}
241 local num_shells=${4:-1}
242 local num_workloads=${5:-1}
243
244 local s=0
245 local w=0
246
247 for i in $(seq $num_iters); do
248
249 for sh_path in $osh_native bash dash; do
250
251 for workload in "${ALL_WORKLOADS[@]}"; do
252 tsv-row $host_name $sh_path $workload
253
254 w=$(( w + 1 )) # cut off at specified workloads
255 if test $w -eq $num_workloads; then
256 break
257 fi
258 done
259
260 s=$(( s + 1 )) # cut off as specified shells
261 if test $s -eq $num_shells; then
262 break
263 fi
264
265 done
266 done
267}
268
269run-tasks-wrapper() {
270 ### reads tasks from stdin
271
272 local host_name=$1 # 'no-host' or 'lenny'
273 local raw_out_dir=$2
274
275 mkdir -v -p $raw_out_dir
276
277 local tsv_out="$raw_out_dir/times.tsv"
278
279 # Write header of the TSV file that is appended to.
280 time-tsv -o $tsv_out --print-header \
281 --rusage \
282 --rusage-2 \
283 --field task_id \
284 --field host_name --field sh_path \
285 --field workload
286
287 # reads tasks from stdin
288 # run-tasks outputs 3 things: raw times.tsv, per-task STDOUT and files, and
289 # per-task GC stats
290 run-tasks $raw_out_dir
291
292 # Turn individual files into a TSV, adding host
293 benchmarks/gc_stats_to_tsv.py $raw_out_dir/gc-*.txt \
294 | tsv-add-const-column host_name "$host_name" \
295 > $raw_out_dir/gc_stats.tsv
296
297 cp -v _tmp/provenance.tsv $raw_out_dir
298}
299
300measure() {
301 ### For release and CI
302 local host_name=$1 # 'no-host' or 'lenny'
303 local raw_out_dir=$2 # _tmp/osh-runtime or ../../benchmark-data/osh-runtime
304 local osh_native=$3 # $OSH_CPP_NINJA_BUILD or $OSH_CPP_BENCHMARK_DATA
305
306 print-tasks "$host_name" "$osh_native" \
307 | run-tasks-wrapper "$host_name" "$raw_out_dir"
308}
309
310stage1() {
311 local base_dir=${1:-$BASE_DIR} # _tmp/osh-runtime or ../benchmark-data/osh-runtime
312 local single_machine=${2:-}
313
314 local out_dir=$BASE_DIR/stage1 # _tmp/osh-runtime
315 mkdir -p $out_dir
316
317 # Globs are in lexicographical order, which works for our dates.
318
319 local -a raw_times=()
320 local -a raw_gc_stats=()
321 local -a raw_provenance=()
322
323 if test -n "$single_machine"; then
324 local -a a=( $base_dir/raw.$single_machine.* )
325
326 raw_times+=( ${a[-1]}/times.tsv )
327 raw_gc_stats+=( ${a[-1]}/gc_stats.tsv )
328 raw_provenance+=( ${a[-1]}/provenance.tsv )
329
330 else
331 local -a a=( $base_dir/raw.$MACHINE1.* )
332 local -a b=( $base_dir/raw.$MACHINE2.* )
333
334 raw_times+=( ${a[-1]}/times.tsv ${b[-1]}/times.tsv )
335 raw_gc_stats+=( ${a[-1]}/gc_stats.tsv ${b[-1]}/gc_stats.tsv )
336 raw_provenance+=( ${a[-1]}/provenance.tsv ${b[-1]}/provenance.tsv )
337 fi
338
339 tsv-concat "${raw_times[@]}" > $out_dir/times.tsv
340
341 tsv-concat "${raw_gc_stats[@]}" > $out_dir/gc_stats.tsv
342
343 tsv-concat "${raw_provenance[@]}" > $out_dir/provenance.tsv
344}
345
346print-report() {
347 local in_dir=$1
348
349 benchmark-html-head 'OSH Runtime Performance'
350
351 cat <<EOF
352 <body class="width60">
353 <p id="home-link">
354 <a href="/">oilshell.org</a>
355 </p>
356EOF
357
358 cmark <<'EOF'
359## OSH Runtime Performance
360
361Source code: [benchmarks/osh-runtime.sh](https://github.com/oilshell/oil/tree/master/benchmarks/osh-runtime.sh)
362
363- [Elapsed Time](#elapsed-time)
364- [Minor Page Faults](#page-faults)
365- [Memory Usage](#memory-usage)
366- [GC Stats](#gc-stats)
367- [rusage Details](#rusage-details)
368- [More Details](#more-details)
369- [Shell and Host](#shell-and-host)
370
371<a name="elapsed-time" />
372
373### Elapsed Time by Shell (milliseconds)
374
375Some benchmarks call many external tools, while some exercise the shell
376interpreter itself.
377EOF
378 tsv2html $in_dir/elapsed.tsv
379
380 cmark <<EOF
381<a name="page-faults" />
382
383### Minor Page Faults
384EOF
385
386 tsv2html $in_dir/page_faults.tsv
387
388 cmark <<EOF
389<a name="memory-usage" />
390
391### Memory Usage (Max Resident Set Size in MB)
392
393Memory usage is measured in MB (powers of 10), not MiB (powers of 2).
394EOF
395 tsv2html $in_dir/max_rss.tsv
396
397 cmark <<EOF
398<a name="gc-stats" />
399
400### GC Stats
401EOF
402 tsv2html $in_dir/gc_stats.tsv
403
404 cmark <<EOF
405<a name="rusage-details" />
406
407### rusage Details
408EOF
409 tsv2html $in_dir/details.tsv
410
411 cmark <<EOF
412<a name="more-details" />
413
414### More Details
415EOF
416 tsv2html $in_dir/details_io.tsv
417
418 cmark <<'EOF'
419<a name="shell-and-host" />
420
421### Shell and Host
422EOF
423 tsv2html $in_dir/shells.tsv
424 tsv2html $in_dir/hosts.tsv
425
426 cmark <<'EOF'
427---
428[raw files](-wwz-index)
429
430 </body>
431</html>
432EOF
433}
434
435test-oils-run() {
436 local osh=$1
437
438 # flags passed by caller
439 local num_iters=${2:-1}
440 local num_shells=${3:-1}
441 local num_workloads=${4:-1}
442
443 local time_py=$XSHAR_DIR/benchmarks/time_.py
444 $time_py --tsv --rusage -- \
445 $osh -c 'echo "smoke test: hi from benchmarks/osh-runtime.sh"'
446
447 local host_name
448 host_name=$(hostname)
449
450 local job_id
451 job_id=$(print-job-id)
452
453 # Write _tmp/provenance.* and _tmp/{host,shell}-id
454 shell-provenance-2 \
455 $host_name $job_id _tmp \
456 bash dash $osh
457
458 # e.g. 2024-05-01__10-11-12.ci-vm-name
459 local raw_out_dir="$BASE_DIR/$job_id.$host_name"
460 mkdir -p $raw_out_dir
461
462 # Similar to 'measure', for soil-run and release
463 print-tasks-xshar $host_name $osh $num_iters $num_shells $num_workloads \
464 | run-tasks-wrapper $host_name $raw_out_dir
465
466 # Note: 'stage1' in soil-run is a trivial concatenation, so we can create input for
467 # benchmarks/report.R. We don't need that here
468}
469
470soil-run() {
471 ### Run it on just this machine, and make a report
472
473 rm -r -f $BASE_DIR
474 mkdir -p $BASE_DIR
475
476 # TODO: This testdata should be baked into Docker image, or mounted
477 download
478 extract
479
480 # could add _bin/cxx-bumpleak/oils-for-unix, although sometimes it's slower
481 local -a osh_bin=( $OSH_CPP_NINJA_BUILD )
482 ninja "${osh_bin[@]}"
483
484 local single_machine='no-host'
485
486 local job_id
487 job_id=$(print-job-id)
488
489 # Write _tmp/provenance.* and _tmp/{host,shell}-id
490 shell-provenance-2 \
491 $single_machine $job_id _tmp \
492 bash dash bin/osh "${osh_bin[@]}"
493
494 local host_job_id="$single_machine.$job_id"
495 local raw_out_dir="$BASE_DIR/raw.$host_job_id"
496 mkdir -p $raw_out_dir $BASE_DIR/stage1
497
498 measure $single_machine $raw_out_dir $OSH_CPP_NINJA_BUILD
499
500 # Trivial concatenation for 1 machine
501 stage1 '' $single_machine
502
503 benchmarks/report.sh stage2 $BASE_DIR
504
505 benchmarks/report.sh stage3 $BASE_DIR
506}
507
508#
509# Debugging
510#
511
512compare-cpython() {
513 #local -a a=( ../benchmark-data/osh-runtime/*.lenny.2024* )
514 local -a a=( ../benchmark-data/osh-runtime/*.hoover.2024* )
515
516 # More of a diff here?
517 #local -a a=( ../benchmark-data/osh-runtime/*.broome.2023* )
518 # less diff here
519 #local -a a=( ../benchmark-data/osh-runtime/*.lenny.2023* )
520
521 local dir=${a[-1]}
522
523 echo $dir
524
525 head -n 1 $dir/times.tsv
526 fgrep 'configure.cpython' $dir/times.tsv
527
528 local bash_id=2
529 local dash_id=8
530 local osh_py_id=14
531 local osh_cpp_id=20
532
533 set +o errexit
534
535 local out_dir=_tmp/cpython-configure
536 mkdir -p $out_dir
537
538 echo 'bash vs. dash'
539 diff -u --recursive $dir/{files-2,files-8} > $out_dir/bash-vs-dash.txt
540 diffstat $out_dir/bash-vs-dash.txt
541 echo
542
543 echo 'bash vs. osh-py'
544 diff -u --recursive $dir/{files-2,files-14} > $out_dir/bash-vs-osh-py.txt
545 diffstat $out_dir/bash-vs-osh-py.txt
546 echo
547
548 echo 'bash vs. osh-cpp'
549 diff -u --recursive $dir/{files-2,files-20} > $out_dir/bash-vs-osh-cpp.txt
550 diffstat $out_dir/bash-vs-osh-cpp.txt
551 echo
552
553 return
554
555 diff -u $dir/{files-2,files-20}/STDOUT.txt
556 echo
557
558 diff -u $dir/{files-2,files-20}/pyconfig.h
559 echo
560
561 cdiff -u $dir/{files-2,files-20}/config.log
562 echo
563}
564
565"$@"