OILS / benchmarks / osh-runtime.sh View on Github | oilshell.org

485 lines, 278 significant
1#!/usr/bin/env bash
2#
3# Test scripts found in the wild for both correctness and performance.
4#
5# Usage:
6# benchmarks/osh-runtime.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13
14source benchmarks/common.sh # tsv-concat
15source benchmarks/id.sh # print-job-id
16source soil/common.sh # find-dir-html
17source test/common.sh
18source test/tsv-lib.sh # tsv-row
19
20readonly BASE_DIR=_tmp/osh-runtime
21
22# TODO: Move to ../oil_DEPS
23readonly TAR_DIR=$PWD/_deps/osh-runtime # Make it absolute
24
25#
26# Dependencies
27#
28
29readonly PY27_DIR=$PWD/Python-2.7.13
30
31# NOTE: Same list in oilshell.org/blob/run.sh.
32tarballs() {
33 cat <<EOF
34tcc-0.9.26.tar.bz2
35yash-2.46.tar.xz
36ocaml-4.06.0.tar.xz
37util-linux-2.40.tar.xz
38EOF
39}
40
41download() {
42 mkdir -p $TAR_DIR
43 tarballs | xargs -n 1 -I {} --verbose -- \
44 wget --no-clobber --directory $TAR_DIR 'https://www.oilshell.org/blob/testdata/{}'
45}
46
47extract() {
48 set -x
49 time for f in $TAR_DIR/*.{bz2,xz}; do
50 tar -x --directory $TAR_DIR --file $f
51 done
52 set +x
53
54 ls -l $TAR_DIR
55}
56
57#
58# Computation
59#
60
61run-tasks() {
62 local raw_out_dir=$1
63 raw_out_dir="$PWD/$raw_out_dir" # because we change dirs
64
65 local task_id=0
66 while read -r host_name sh_path workload; do
67
68 log "*** $host_name $sh_path $workload $task_id"
69
70 local sh_run_path
71 case $sh_path in
72 /*) # Already absolute
73 sh_run_path=$sh_path
74 ;;
75 */*) # It's relative, so make it absolute
76 sh_run_path=$PWD/$sh_path
77 ;;
78 *) # 'dash' should remain 'dash'
79 sh_run_path=$sh_path
80 ;;
81 esac
82
83 local working_dir=''
84 local files_out_dir="$raw_out_dir/files-$task_id"
85 mkdir -v -p $files_out_dir
86
87 local save_new_files=''
88
89 local -a argv
90 case $workload in
91 hello-world)
92 argv=( testdata/osh-runtime/hello_world.sh )
93 ;;
94
95 bin-true)
96 argv=( testdata/osh-runtime/bin_true.sh )
97 ;;
98
99 abuild-print-help)
100 argv=( testdata/osh-runtime/abuild -h )
101 ;;
102
103 configure.cpython)
104 argv=( $PY27_DIR/configure )
105 working_dir=$files_out_dir
106 ;;
107
108 configure.util-linux)
109 # flag needed to avoid sqlite3 dep error message
110 argv=( $TAR_DIR/util-linux-2.40/configure --disable-liblastlog2 )
111 working_dir=$files_out_dir
112 ;;
113
114 configure.*)
115 argv=( ./configure )
116
117 local conf_dir
118 case $workload in
119 *.ocaml)
120 conf_dir='ocaml-4.06.0'
121 ;;
122 *.tcc)
123 conf_dir='tcc-0.9.26'
124 ;;
125 *.yash)
126 conf_dir='yash-2.46'
127 ;;
128 *)
129 die "Invalid workload $workload"
130 esac
131
132 # These are run in-tree?
133 working_dir=$TAR_DIR/$conf_dir
134 ;;
135
136 *)
137 die "Invalid workload $workload"
138 ;;
139 esac
140
141 local -a time_argv=(
142 time-tsv
143 --output "$raw_out_dir/times.tsv" --append
144 --rusage
145 --rusage-2
146 --field "$task_id"
147 --field "$host_name" --field "$sh_path"
148 --field "$workload"
149 -- "$sh_run_path" "${argv[@]}"
150 )
151
152 local stdout_file="$files_out_dir/STDOUT.txt"
153 local gc_stats_file="$raw_out_dir/gc-$task_id.txt"
154
155 # Maybe change dirs
156 if test -n "$working_dir"; then
157 pushd "$working_dir"
158 fi
159
160 if test -n "$save_new_files"; then
161 touch __TIMESTAMP
162 fi
163
164 # Run it, possibly with GC stats
165 case $sh_path in
166 *_bin/*/osh)
167 OILS_GC_STATS_FD=99 "${time_argv[@]}" > $stdout_file 99> $gc_stats_file
168 ;;
169 *)
170 "${time_argv[@]}" > $stdout_file
171 ;;
172 esac
173
174 if test -n "$save_new_files"; then
175 echo "COPYING to $files_out_dir"
176 find . -type f -newer __TIMESTAMP \
177 | xargs -I {} -- cp --verbose {} $files_out_dir
178 fi
179
180 # Restore dir
181 if test -n "$working_dir"; then
182 popd
183 fi
184
185 task_id=$((task_id + 1))
186 done
187}
188
189print-tasks() {
190 local host_name=$1
191 local osh_native=$2
192
193 local -a workloads=(
194 hello-world
195 bin-true
196 abuild-print-help
197
198 configure.cpython
199 configure.util-linux
200 configure.ocaml
201 configure.tcc
202 configure.yash
203 )
204
205 if test -n "${QUICKLY:-}"; then
206 # Just do the first two
207 workloads=(
208 #configure.util-linux
209 hello-world
210 bin-true
211 #abuild-print-help
212 )
213 fi
214
215 for sh_path in bash dash bin/osh $osh_native; do
216 for workload in "${workloads[@]}"; do
217 tsv-row $host_name $sh_path $workload
218 done
219 done
220}
221
222measure() {
223 local host_name=$1 # 'no-host' or 'lenny'
224 local raw_out_dir=$2
225 local osh_native=$3 # $OSH_CPP_NINJA_BUILD or $OSH_CPP_BENCHMARK_DATA
226 local out_dir=${4:-$BASE_DIR} # ../benchmark-data/osh-runtime or _tmp/osh-runtime
227
228 mkdir -v -p $raw_out_dir
229
230 local tsv_out="$raw_out_dir/times.tsv"
231
232 # Write header of the TSV file that is appended to.
233 time-tsv -o $tsv_out --print-header \
234 --rusage \
235 --rusage-2 \
236 --field task_id \
237 --field host_name --field sh_path \
238 --field workload
239
240 # run-tasks outputs 3 things: raw times.tsv, per-task STDOUT and files, and
241 # per-task GC stats
242 print-tasks $host_name $osh_native | run-tasks $raw_out_dir
243
244 # Turn individual files into a TSV, adding host
245 benchmarks/gc_stats_to_tsv.py $raw_out_dir/gc-*.txt \
246 | tsv-add-const-column host_name "$host_name" \
247 > $raw_out_dir/gc_stats.tsv
248
249 cp -v _tmp/provenance.tsv $raw_out_dir
250}
251
252stage1() {
253 local base_dir=${1:-$BASE_DIR} # _tmp/osh-runtime or ../benchmark-data/osh-runtime
254 local single_machine=${2:-}
255
256 local out_dir=$BASE_DIR/stage1 # _tmp/osh-runtime
257 mkdir -p $out_dir
258
259 # Globs are in lexicographical order, which works for our dates.
260
261 local -a raw_times=()
262 local -a raw_gc_stats=()
263 local -a raw_provenance=()
264
265 if test -n "$single_machine"; then
266 local -a a=( $base_dir/raw.$single_machine.* )
267
268 raw_times+=( ${a[-1]}/times.tsv )
269 raw_gc_stats+=( ${a[-1]}/gc_stats.tsv )
270 raw_provenance+=( ${a[-1]}/provenance.tsv )
271
272 else
273 local -a a=( $base_dir/raw.$MACHINE1.* )
274 local -a b=( $base_dir/raw.$MACHINE2.* )
275
276 raw_times+=( ${a[-1]}/times.tsv ${b[-1]}/times.tsv )
277 raw_gc_stats+=( ${a[-1]}/gc_stats.tsv ${b[-1]}/gc_stats.tsv )
278 raw_provenance+=( ${a[-1]}/provenance.tsv ${b[-1]}/provenance.tsv )
279 fi
280
281 tsv-concat "${raw_times[@]}" > $out_dir/times.tsv
282
283 tsv-concat "${raw_gc_stats[@]}" > $out_dir/gc_stats.tsv
284
285 tsv-concat "${raw_provenance[@]}" > $out_dir/provenance.tsv
286}
287
288print-report() {
289 local in_dir=$1
290
291 benchmark-html-head 'OSH Runtime Performance'
292
293 cat <<EOF
294 <body class="width60">
295 <p id="home-link">
296 <a href="/">oilshell.org</a>
297 </p>
298EOF
299
300 cmark <<'EOF'
301## OSH Runtime Performance
302
303Source code: [benchmarks/osh-runtime.sh](https://github.com/oilshell/oil/tree/master/benchmarks/osh-runtime.sh)
304
305- [Elapsed Time](#elapsed-time)
306- [Minor Page Faults](#page-faults)
307- [Memory Usage](#memory-usage)
308- [GC Stats](#gc-stats)
309- [rusage Details](#rusage-details)
310- [More Details](#more-details)
311- [Shell and Host](#shell-and-host)
312
313<a name="elapsed-time" />
314
315### Elapsed Time by Shell (milliseconds)
316
317Some benchmarks call many external tools, while some exercise the shell
318interpreter itself.
319EOF
320 tsv2html $in_dir/elapsed.tsv
321
322 cmark <<EOF
323<a name="page-faults" />
324
325### Minor Page Faults
326EOF
327
328 tsv2html $in_dir/page_faults.tsv
329
330 cmark <<EOF
331<a name="memory-usage" />
332
333### Memory Usage (Max Resident Set Size in MB)
334
335Memory usage is measured in MB (powers of 10), not MiB (powers of 2).
336EOF
337 tsv2html $in_dir/max_rss.tsv
338
339 cmark <<EOF
340<a name="gc-stats" />
341
342### GC Stats
343EOF
344 tsv2html $in_dir/gc_stats.tsv
345
346 cmark <<EOF
347<a name="rusage-details" />
348
349### rusage Details
350EOF
351 tsv2html $in_dir/details.tsv
352
353 cmark <<EOF
354<a name="more-details" />
355
356### More Details
357EOF
358 tsv2html $in_dir/details_io.tsv
359
360 cmark <<'EOF'
361<a name="shell-and-host" />
362
363### Shell and Host
364EOF
365 tsv2html $in_dir/shells.tsv
366 tsv2html $in_dir/hosts.tsv
367
368 # Only show files.html link on a single machine
369 if test -f $(dirname $in_dir)/files.html; then
370 cmark <<'EOF'
371---
372
373[raw files](files.html)
374EOF
375 fi
376
377 cat <<EOF
378 </body>
379</html>
380EOF
381}
382
383test-oils-run() {
384 echo 'Hello from benchmarks/osh-runtime.sh'
385}
386
387soil-run() {
388 ### Run it on just this machine, and make a report
389
390 rm -r -f $BASE_DIR
391 mkdir -p $BASE_DIR
392
393 # TODO: This testdata should be baked into Docker image, or mounted
394 download
395 extract
396
397 # could add _bin/cxx-bumpleak/oils-for-unix, although sometimes it's slower
398 local -a osh_bin=( $OSH_CPP_NINJA_BUILD )
399 ninja "${osh_bin[@]}"
400
401 local single_machine='no-host'
402
403 local job_id
404 job_id=$(print-job-id)
405
406 # Write _tmp/provenance.* and _tmp/{host,shell}-id
407 shell-provenance-2 \
408 $single_machine $job_id _tmp \
409 bash dash bin/osh "${osh_bin[@]}"
410
411 local host_job_id="$single_machine.$job_id"
412 local raw_out_dir="$BASE_DIR/raw.$host_job_id"
413 mkdir -p $raw_out_dir $BASE_DIR/stage1
414
415 measure $single_machine $raw_out_dir $OSH_CPP_NINJA_BUILD
416
417 # Trivial concatenation for 1 machine
418 stage1 '' $single_machine
419
420 benchmarks/report.sh stage2 $BASE_DIR
421
422 # Make _tmp/osh-parser/files.html, so index.html can potentially link to it
423 find-dir-html _tmp/osh-runtime files
424
425 benchmarks/report.sh stage3 $BASE_DIR
426}
427
428#
429# Debugging
430#
431
432compare-cpython() {
433 #local -a a=( ../benchmark-data/osh-runtime/*.lenny.2024* )
434 local -a a=( ../benchmark-data/osh-runtime/*.hoover.2024* )
435
436 # More of a diff here?
437 #local -a a=( ../benchmark-data/osh-runtime/*.broome.2023* )
438 # less diff here
439 #local -a a=( ../benchmark-data/osh-runtime/*.lenny.2023* )
440
441 local dir=${a[-1]}
442
443 echo $dir
444
445 head -n 1 $dir/times.tsv
446 fgrep 'configure.cpython' $dir/times.tsv
447
448 local bash_id=2
449 local dash_id=8
450 local osh_py_id=14
451 local osh_cpp_id=20
452
453 set +o errexit
454
455 local out_dir=_tmp/cpython-configure
456 mkdir -p $out_dir
457
458 echo 'bash vs. dash'
459 diff -u --recursive $dir/{files-2,files-8} > $out_dir/bash-vs-dash.txt
460 diffstat $out_dir/bash-vs-dash.txt
461 echo
462
463 echo 'bash vs. osh-py'
464 diff -u --recursive $dir/{files-2,files-14} > $out_dir/bash-vs-osh-py.txt
465 diffstat $out_dir/bash-vs-osh-py.txt
466 echo
467
468 echo 'bash vs. osh-cpp'
469 diff -u --recursive $dir/{files-2,files-20} > $out_dir/bash-vs-osh-cpp.txt
470 diffstat $out_dir/bash-vs-osh-cpp.txt
471 echo
472
473 return
474
475 diff -u $dir/{files-2,files-20}/STDOUT.txt
476 echo
477
478 diff -u $dir/{files-2,files-20}/pyconfig.h
479 echo
480
481 cdiff -u $dir/{files-2,files-20}/config.log
482 echo
483}
484
485"$@"