OILS / soil / worker.sh View on Github | oilshell.org

639 lines, 208 significant
1#!/usr/bin/env bash
2#
3# Run continuous build tasks.
4#
5# Usage:
6# soil/worker.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd) # tsv-lib.sh uses this
13readonly REPO_ROOT
14
15source soil/common.sh
16source test/tsv-lib.sh
17
18py-all-and-ninja() {
19 ### baseline for most tasks
20
21 build/py.sh all
22 ./NINJA-config.sh
23}
24
25ninja-config() {
26 ./NINJA-config.sh
27}
28
29dummy-tasks() {
30 ### Print tasks that execute quickly
31
32 # (task_name, script, action, result_html)
33 cat <<EOF
34os-info soil/diagnose.sh os-info -
35dump-env soil/diagnose.sh dump-env -
36EOF
37}
38
39raw-vm-tasks() {
40 # The perf tool depends on a specific version of a kernel, so run it outside
41 # a container.
42
43 # (task_name, script, action, result_html)
44 cat <<EOF
45os-info soil/diagnose.sh os-info -
46dump-env soil/diagnose.sh dump-env -
47perf-install benchmarks/perf.sh soil-install -
48wait-for-tarball soil/wait.sh for-cpp-tarball -
49test-install-tar devtools/release-native.sh test-install-tar -
50perf-profiles benchmarks/perf.sh soil-run _tmp/perf/index.html
51EOF
52}
53
54# Oh there is a large list of pre-installed software
55# https://github.com/actions/runner-images#available-images
56# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2004-Readme.md
57# https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
58#
59# 1. System deps for building wedges - ninja, cmake, libreadline-dev, etc.
60# 2. fetch wedges - re2c, cmark, python2, python3, MyPy, pyflakes
61# - Python 3.10 desired for "pea"
62# 3. build them
63# 4. build Oils with them
64
65dev-setup-for() {
66 local distro=$1
67 # (task_name, script, action, result_html)
68
69 cat <<EOF
70os-info soil/diagnose.sh os-info -
71dump-env soil/diagnose.sh dump-env -
72wedge-deps build/deps.sh wedge-deps-$distro -
73fetch build/deps.sh fetch -
74install-wedges build/deps.sh install-wedges-soil _build/wedge/logs/index.html
75py-all-and-ninja soil/worker.sh py-all-and-ninja -
76smoke-test build/dev-setup-test.sh smoke-test -
77wedge-report build/deps.sh wedge-report -
78EOF
79}
80
81spec-bin-for() {
82 local distro=$1
83 cat <<EOF
84os-info soil/diagnose.sh os-info -
85dump-env soil/diagnose.sh dump-env -
86wedge-deps build/deps.sh wedge-deps-$distro -
87fetch build/deps.sh fetch -
88spec-bin build/deps.sh install-spec-bin-fast _build/wedge/logs/index.html
89EOF
90}
91
92dev-setup-debian-tasks() {
93 # (task_name, script, action, result_html)
94
95 dev-setup-for debian
96 #spec-bin-for debian
97}
98
99dev-setup-fedora-tasks() {
100 # (task_name, script, action, result_html)
101
102 dev-setup-for fedora
103}
104
105dev-setup-alpine-tasks() {
106 # (task_name, script, action, result_html)
107
108 dev-setup-for alpine
109 #spec-bin-for alpine
110}
111
112pea-tasks() {
113 ### Print tasks for the 'pea' build
114
115 # We need a later version of Python 3 / MyPy both to type check and
116 # to parse
117
118 # Run py-source so we can type check generated code
119 # We need to type check more than we translate
120
121 # (task_name, script, action, result_html)
122 cat <<EOF
123os-info soil/diagnose.sh os-info -
124dump-env soil/diagnose.sh dump-env -
125py-source build/py.sh py-source -
126check-types pea/TEST.sh check-types -
127run-tests pea/TEST.sh run-tests -
128parse-all pea/TEST.sh parse-all -
129EOF
130}
131
132dev-minimal-tasks() {
133 ### Print tasks for the 'dev-minimal' build
134
135 # repo overview is suggested by README.md
136
137 # (task_name, script, action, result_html)
138 cat <<EOF
139os-info soil/diagnose.sh os-info -
140dump-env soil/diagnose.sh dump-env -
141build-minimal build/py.sh minimal -
142repo-overview metrics/source-code.sh overview -
143lint test/lint.sh soil-run -
144asdl-types asdl/TEST.sh check-types -
145oil-types devtools/types.sh soil-run -
146unit test/unit.sh minimal _test/py-unit/
147lossless test/lossless.sh soil-run -
148parse-errors test/parse-errors.sh soil-run-py -
149runtime-errors test/runtime-errors.sh soil-run-py -
150ysh-parse-errors test/ysh-parse-errors.sh soil-run-py -
151ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-py -
152ysh-every-string test/ysh-every-string.sh soil-run-py -
153ysh-large ysh/run.sh soil-run -
154j8-errors data_lang/j8-errors.sh soil-run-py -
155link-busybox-ash test/spec-bin.sh link-busybox-ash -
156osh-minimal test/spec-py.sh osh-minimal _tmp/spec/osh-minimal/index.html
157headless client/run.sh soil-run-py -
158EOF
159}
160
161interactive-tasks() {
162 ### Print tasks for the 'interactive' build
163
164 # TODO: also run interactive suite with osh-cpp
165
166# TODO: Why does the needs-terminal suite hang in Docker? It doesn't hang in an interactive Docker session.
167#
168# needs-terminal test/spec-py.sh needs-terminal-all _tmp/spec/needs-terminal-py/index.html
169#
170# https://oilshell.zulipchat.com/#narrow/stream/121539-oil-dev/topic/Spec.20Tests.20for.20Interactive.20Parsing
171
172 cat <<EOF
173os-info soil/diagnose.sh os-info -
174dump-env soil/diagnose.sh dump-env -
175py-all-and-ninja soil/worker.sh py-all-and-ninja -
176interactive-osh test/spec-py.sh interactive-osh _tmp/spec/interactive-osh/index.html
177nohup test/nohup.sh soil-run -
178process-table test/process-table.sh soil-run _tmp/process-table/index.html
179stateful test/stateful.sh soil-run _tmp/spec/stateful/index.html
180EOF
181
182}
183
184wild-tasks() {
185 ### Print tasks for the 'wild' build
186
187 # (task_name, script, action, result_html)
188 cat <<EOF
189os-info soil/diagnose.sh os-info -
190dump-env soil/diagnose.sh dump-env -
191wait-for-tarball soil/wait.sh for-cpp-tarball -
192test-tar devtools/release-native.sh test-tar -
193linecount metrics/tarball.sh linecount-oils-cpp -
194wild test/wild.sh soil-run _tmp/wild-www/index.html
195EOF
196}
197
198benchmarks-tasks() {
199 # (task_name, script, action, result_html)
200
201 cat <<EOF
202os-info soil/diagnose.sh os-info -
203dump-env soil/diagnose.sh dump-env -
204py-all-and-ninja soil/worker.sh py-all-and-ninja -
205dev-shell-test build/dev-shell-test.sh soil-run -
206id-test benchmarks/id-test.sh soil-run -
207osh-parser benchmarks/osh-parser.sh soil-run _tmp/osh-parser/index.html
208osh-runtime benchmarks/osh-runtime.sh soil-run _tmp/osh-runtime/index.html
209vm-baseline benchmarks/vm-baseline.sh soil-run _tmp/vm-baseline/index.html
210compute benchmarks/compute.sh soil-run _tmp/compute/index.html
211gc benchmarks/gc.sh soil-run _tmp/gc/index.html
212mycpp-benchmarks benchmarks/mycpp.sh soil-run _tmp/mycpp-examples/-wwz-index
213EOF
214}
215
216bloaty-tasks() {
217 cat <<EOF
218os-info soil/diagnose.sh os-info -
219dump-env soil/diagnose.sh dump-env -
220wait-for-tarball soil/wait.sh for-cpp-tarball -
221test-tar devtools/release-native.sh test-tar -
222native-code metrics/native-code.sh oils-for-unix _tmp/metrics/oils-for-unix/
223EOF
224}
225
226benchmarks2-tasks() {
227 # Note: id-test doesn't run in 'other-tests' because 'gawk' isn't in that image
228 cat <<EOF
229os-info soil/diagnose.sh os-info -
230dump-env soil/diagnose.sh dump-env -
231wait-for-tarball soil/wait.sh for-cpp-tarball -
232test-tar devtools/release-native.sh test-tar -
233uftrace benchmarks/uftrace.sh soil-run _tmp/uftrace/index.html
234gc-cachegrind benchmarks/gc-cachegrind.sh soil-run _tmp/gc-cachegrind/index.html
235EOF
236}
237
238cpp-spec-tasks() {
239 # (task_name, script, action, result_html)
240
241 cat <<EOF
242os-info soil/diagnose.sh os-info -
243dump-env soil/diagnose.sh dump-env -
244py-all-and-ninja soil/worker.sh py-all-and-ninja -
245oils-cpp-smoke build/native.sh soil-run -
246osh-all test/spec-cpp.sh osh-all _tmp/spec/osh-cpp/compare.html
247ysh-all test/spec-cpp.sh ysh-all _tmp/spec/ysh-cpp/compare.html
248ysh-py test/spec-py.sh ysh-all-serial _tmp/spec/ysh-py/index.html
249EOF
250}
251
252cpp-tarball-tasks() {
253
254 # Note: build-times task requires _build/oils.sh
255 # It's a bit redundant with test-tar
256
257 cat <<EOF
258os-info soil/diagnose.sh os-info -
259dump-env soil/diagnose.sh dump-env -
260py-all-and-ninja soil/worker.sh py-all-and-ninja -
261oils-cpp-smoke build/native.sh soil-run -
262make-tar devtools/release-native.sh make-tar _release/oils-for-unix.tar
263xshar-hello devtools/xshar.sh soil-run-hello _release/hello-xshar.xshar
264xshar-test-oils devtools/xshar.sh soil-run-test-oils _release/test-oils.xshar
265build-times build/native.sh measure-build-times -
266EOF
267
268# build-times is a good enough test
269# test-tar devtools/release-native.sh test-tar -
270#
271# Note: tarball is deployed outside the container
272
273}
274
275cpp-small-tasks() {
276
277 # yaks could be moved to pea/ image once it has python2-dev
278 cat <<EOF
279os-info soil/diagnose.sh os-info -
280dump-env soil/diagnose.sh dump-env -
281py-all-and-ninja soil/worker.sh py-all-and-ninja -
282py-unit test/unit.sh all _test/py-unit/
283yaks yaks/TEST.sh soil-run -
284oils-cpp-smoke build/native.sh soil-run -
285cpp-unit test/cpp-unit.sh soil-run _test/-wwz-index
286headless client/run.sh soil-run-cpp -
287asan test/asan.sh soil-run -
288ltrace test/ltrace.sh soil-run -
289micro-syntax doctools/micro-syntax.sh soil-run -
290src-tree doctools/src-tree.sh soil-run _tmp/src-tree-www/index.html
291line-counts metrics/source-code.sh write-reports _tmp/metrics/line-counts/-wwz-index
292preprocessed metrics/source-code.sh preprocessed _tmp/metrics/preprocessed/-wwz-index
293mycpp-examples mycpp/TEST.sh soil-run _test/-wwz-index
294parse-errors test/parse-errors.sh soil-run-cpp -
295runtime-errors test/runtime-errors.sh soil-run-cpp -
296ysh-parse-errors test/ysh-parse-errors.sh soil-run-cpp -
297ysh-runtime-errors test/ysh-runtime-errors.sh soil-run-cpp -
298ysh-every-string test/ysh-every-string.sh soil-run-cpp -
299ysh-large ysh/run.sh soil-run-cpp -
300j8-errors data_lang/j8-errors.sh soil-run-cpp -
301houston-fp demo/houston-fp/run.sh soil-run -
302souffle-smoke-test test/souffle-smoke.sh soil-run -
303EOF
304}
305
306cpp-coverage-tasks() {
307 # dep notes: hnode_asdl.h required by expr_asdl.h in mycpp/examples
308
309 cat <<EOF
310os-info soil/diagnose.sh os-info -
311dump-env soil/diagnose.sh dump-env -
312py-all-and-ninja soil/worker.sh py-all-and-ninja -
313extract-clang deps/from-binary.sh extract-clang-in-container -
314mycpp-unit-coverage mycpp/TEST.sh unit-test-coverage _test/clang-coverage/mycpp/html/index.html
315mycpp-examples-coverage mycpp/TEST.sh examples-coverage _test/clang-coverage/mycpp/examples/html/index.html
316cpp-coverage cpp/TEST.sh coverage _test/clang-coverage/cpp/html/index.html
317unified-coverage test/coverage.sh unified-report _test/clang-coverage/unified/html/index.html
318compare-gcc-clang metrics/native-code.sh compare-gcc-clang _tmp/metrics/compare-gcc-clang.txt
319EOF
320}
321
322ovm-tarball-tasks() {
323 ### Print tasks for the 'ovm-tarball' build
324
325 # notes:
326 # - py-all needed to crawl dependencies to make tarball.
327
328 # (task_name, script, action, result_html)
329 cat <<EOF
330os-info soil/diagnose.sh os-info -
331dump-env soil/diagnose.sh dump-env -
332py-all build/py.sh all -
333syscall-by-code test/syscall.sh by-code _tmp/syscall/by-code.txt
334syscall-by-input test/syscall.sh by-input _tmp/syscall/by-input.txt
335osh-spec test/spec-py.sh osh-all-serial _tmp/spec/osh-py/index.html
336gold test/gold.sh soil-run -
337osh-usage test/osh-usage.sh soil-run -
338tools-deps test/tools-deps.sh soil-run -
339make-tarball devtools/release.sh py-tarball _release/oil.tar
340ysh-ovm-tarball test/spec-py.sh ysh-ovm-tarball _tmp/spec/ysh-py/index.html
341docs build/doc.sh soil-run _release/VERSION/index.html
342doc-metrics echo no-op _release/VERSION/doc/metrics.txt
343EOF
344
345# doc-metrics is a no-op, just for the link. Because soil-run just runs the
346# release, which creates metrics.
347}
348
349# Reuse ovm-tarball container
350app-tests-tasks() {
351
352 cat <<EOF
353os-info soil/diagnose.sh os-info -
354dump-env soil/diagnose.sh dump-env -
355py-all build/py.sh all -
356ble-clone test/ble.sh clone -
357ble-build test/ble.sh build -
358ble-bash-suite test/ble.sh bash-suite -
359ble-test-osh-py test/ble.sh run-tests-osh-py -
360wait-for-tarball soil/wait.sh for-cpp-tarball -
361test-tar devtools/release-native.sh test-tar -
362ble-test-osh-cpp test/ble.sh run-tests-osh-cpp -
363EOF
364
365# This doesn't work
366# ble-test-osh-bash test/ble.sh run-tests-osh-bash -
367}
368
369# TODO: Most of these should be Ninja tasks.
370# Other tests:
371# find-test, xargs-test, pgen2-test, web/table/csv2html-test.sh
372# Probably should start using a shell test framework too.
373other-tests-tasks() {
374 cat <<EOF
375os-info soil/diagnose.sh os-info -
376dump-env soil/diagnose.sh dump-env -
377build-minimal build/py.sh minimal -
378configure-test ./configure-test.sh soil_run -
379time-test benchmarks/time-test.sh soil-run -
380tsv-lib-test test/tsv-lib-test.sh soil-run -
381ysh-ify test/ysh-ify.sh soil-run -
382R-test devtools/R-test.sh soil-run -
383xargs-test test/other.sh xargs-test -
384csv2html-test test/other.sh csv2html-test -
385oil-python-symbols metrics/source-code.sh oil-python-symbols -
386opyc test/opyc.sh soil-run -
387opy-count-lines opy/soil.sh count-lines -
388test-gold opy/soil.sh test-gold -
389build-oil-repo opy/soil.sh build-oil-repo -
390regtest-compile opy/soil.sh regtest-compile -
391EOF
392}
393
394tests-todo() {
395 ### More tests to add
396 find . -name '_*' -a -prune -o -name '*-test.sh' -a -print
397
398 # pgen2/pgen2-test.sh seems mostly broken
399}
400
401# Redefinition for quicker cloud debugging
402maybe-merge-tasks() {
403 cat <<EOF
404os-info soil/diagnose.sh os-info -
405dump-env soil/diagnose.sh dump-env -
406maybe-merge soil/maybe-merge.sh soil-run -
407EOF
408}
409
410run-tasks() {
411 ### Run the tasks on stdin and write _tmp/soil/INDEX.tsv.
412 local job_name=$1
413 local out_dir=$2 # should already exist
414 local tty=$3
415
416 mkdir -p $out_dir/logs
417
418 # So we can run benchmarks/time_.py.
419 # 2023-02-28: Images like soil-wild based off the new soil-common no longer
420 # have 'cc'. Instead they use a wedge.
421 if command -v cc > /dev/null; then
422 build/py.sh time-helper
423 else
424 echo 'test time-tsv3'
425 time-tsv3 -o /tmp/echo.tsv --append -- echo hi
426
427 echo '/tmp/echo.tsv:'
428 cat /tmp/echo.tsv
429 fi
430
431 # For the later deploy step to pick up
432 date +%s > $out_dir/task-run-start-time.txt
433
434 # This data can go on the dashboard index
435 local tsv=$out_dir/INDEX.tsv
436 rm -f $tsv
437
438 local status
439 local max_status=0
440
441 while read task_name script action result_html; do
442 log "--- task: $task_name ---"
443
444 local log_path=$out_dir/logs/$task_name.txt
445
446 # 15 minutes per task
447 # One of the longest tasks is test/spec-cpp, which takes around 420 seconds
448 # TODO: should have a configurable timeout
449 local -a timeout=(timeout 900)
450 local stdin_tty=''
451
452 case $script in
453 test/process-table.sh)
454 # Workaround for weird interaction, see
455 # $ test/process-table.sh timeout-issue
456 timeout=()
457 ;;
458 test/nohup.sh)
459 # Only run test/nohup.sh with TTY. For some reason build/py.sh all hangs
460 # with $tty?
461 stdin_tty=$tty
462 ;;
463 esac
464
465 local -a argv=(
466 time-tsv3 -o $tsv --append
467 --field $task_name --field $script --field $action
468 --field $result_html --
469 "${timeout[@]}" "$script" "$action"
470 )
471
472 # Run task and save status
473 set +o errexit
474 if test -n "$stdin_tty"; then
475 # explicitly connect TTY, e.g. for soil/interactive
476 "${argv[@]}" > $log_path 2>&1 < $stdin_tty
477 else
478 "${argv[@]}" > $log_path 2>&1
479 fi
480 status=$?
481 set -o errexit
482
483 if test "$status" -gt "$max_status"; then
484 max_status=$status
485 fi
486
487 # Show the last line
488 echo
489 tsv-row status elapsed task script action result_html
490 tail -n 1 $tsv
491 echo
492 log "status=$status max_status=$max_status"
493 done
494
495 log '--- done ---'
496 ls -l $out_dir
497 wc -l $out_dir/logs/*
498
499 # This suppressed the deployment of logs, which we don't want. So all our
500 # Travis builds succeed? But then we can't use their failure notifications
501 # (which might be OK).
502 if false; then
503 # exit with the maximum exit code.
504 awk '
505 BEGIN { max = 0 }
506 { if ($1 > max) { max = $1 } }
507 END { exit(max) }
508 ' $tsv
509 fi
510
511 # To fail later. Important: this dir persists across jobs; it's NOT removed
512 # by 'host-shim.sh job-reset'.
513 mkdir -p _soil-jobs
514
515 # Hack: Assign job_id and write it to the status file. Other jobs can poll
516 # for completion of this job and access its resources.
517
518 local job_id
519 job_id="$(date +%Y-%m-%d__%H-%M-%S)"
520
521 # e.g. _soil-jobs/dummy.status.txt
522 echo "$max_status $job_id" > _soil-jobs/$job_name.status.txt
523}
524
525save-metadata() {
526 ### Write metadata files to be saved as JSON
527
528 # NOTE: host-shim.sh also writes image-pull-time.txt
529
530 local job_name=$1
531 local meta_dir=$2
532
533 echo "$job_name" > $meta_dir/job-name.txt
534
535 # command to show current branch
536 # This does NOT work in detached HEAD! Travis puts the branch in an env
537 # variable, but sr.ht doesn't.
538 # git rev-parse --abbrev-ref HEAD > $meta_dir/git-branch.txt
539
540 git log -n 1 --pretty='format:%H' > $meta_dir/commit-hash.txt
541
542 # ISO 8601 format
543 # Note: this can get confused with rebases. Two different commits can have
544 # the same date.
545 git log -n 1 --pretty='format:%aI' > $meta_dir/commit-date.txt
546
547 git log -n 1 --pretty='format:%s' > $meta_dir/commit-line.txt # "subject"
548}
549
550disable-git-errors() {
551
552 # 2023-02: The build started failing because of the permissions we set in
553 # soil/host-shim.sh mount-perms.
554 #
555 # The issue is that the guest needs to be able to write to the Docker mount
556 # of the repo. I think it may have been related to podman vs. Docker.
557 # Should check if mount-perms is necessary in both places.
558 #
559 # git fails unless we have this workaround.
560
561 # https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-repositor
562
563 # https://github.blog/2022-04-12-git-security-vulnerability-announced/
564
565 #git config --global --add safe.directory '*'
566
567 git config --global --add safe.directory /home/uke/oil
568}
569
570job-main() {
571 local job_name=$1
572
573 local out_dir=_tmp/soil
574
575 # Report for debugging
576 export EXTRA_MYCPP_ARGS='--stack-roots-warn 16'
577
578 log-context 'job-main'
579 mkdir -v -p $out_dir
580 ls -l -d $out_dir
581
582 disable-git-errors
583
584 save-metadata $job_name $out_dir
585
586 local captured
587
588 set +o errexit
589 captured=$(tty)
590 status=$?
591 set -o errexit
592
593 if test $status -eq 0; then
594 echo "TTY = $captured"
595 local tty=$captured
596 else
597 echo "captured = $captured"
598 local tty='' # clear the output
599 fi
600 echo
601
602 ${job_name}-tasks | run-tasks $job_name $out_dir "$tty"
603}
604
605JOB-dummy() { job-main 'dummy'; }
606JOB-raw-vm() { job-main 'raw-vm'; }
607JOB-dev-setup-debian() { job-main 'dev-setup-debian'; }
608JOB-dev-setup-fedora() { job-main 'dev-setup-fedora'; }
609JOB-dev-setup-alpine() { job-main 'dev-setup-alpine'; }
610
611JOB-dev-minimal() { job-main 'dev-minimal'; }
612JOB-interactive() { job-main 'interactive'; }
613
614JOB-other-tests() { job-main 'other-tests'; }
615
616JOB-pea() { job-main 'pea'; }
617
618JOB-ovm-tarball() { job-main 'ovm-tarball'; }
619JOB-app-tests() { job-main 'app-tests'; }
620
621JOB-cpp-coverage() { job-main 'cpp-coverage'; }
622JOB-cpp-small() { job-main 'cpp-small'; }
623JOB-cpp-tarball() { job-main 'cpp-tarball'; }
624JOB-cpp-spec() { job-main 'cpp-spec'; }
625
626JOB-bloaty() { job-main 'bloaty'; }
627JOB-benchmarks() { job-main 'benchmarks'; }
628JOB-benchmarks2() { job-main 'benchmarks2'; }
629
630JOB-wild() { job-main 'wild'; }
631
632JOB-maybe-merge() { job-main 'maybe-merge'; }
633
634list-jobs() {
635 # dev-setup-fedora for Fedora, disable
636 compgen -A function | grep -- '^JOB-' | sed 's/^JOB-//g' | egrep -v 'maybe-merge|dev-setup-fedora|dev-setup-alpine'
637}
638
639"$@"