OILS / devtools / release.sh View on Github | oilshell.org

817 lines, 389 significant
1#!/usr/bin/env bash
2#
3# The big Oils release process.
4#
5# Usage:
6# devtools/release.sh <function name>
7#
8# Steps:
9# edit oil-version.txt, build/doc.sh update-src-versions, bump devtools/release-note.sh
10# $0 make-release-branch
11# $0 two-tarballs # CPython, then oils-for-unix, which is INSTALLED
12# demo/osh-debug.sh osh-for-release: Start a shell to dogfood
13# build/cpython-defs.sh {oil-py-names,filter-methods}
14# (regenerate C source)
15#
16# Run on each machine:
17# $0 auto-machine1
18# $0 auto-machine2 ($0 dep-benchmarks first)
19#
20# In between:
21# [switch benchmarks-data repo] commit src/oil-for-unix-* and push to flanders.
22# TODO: Make sure benchmark-data directory is clean!
23#
24# Resume manual work
25#
26# Commit files to oilshell/benchmark-data repo and sync.
27# benchmarks/report.sh all
28# $0 deploy-tar # needed to publish tarball checksum in HTML
29# build/doc.sh run-for-release
30# $0 compress
31# devtools/release-version.sh git-changelog-$VERSION
32# devtools/release-version.sh announcement-$VERSION
33# MAYBE: ./local.sh test-release-tree if you want to preview it
34# $0 deploy-doc (makes releases.html)
35#
36# demo/osh-debug.sh analyze # see what you ran
37#
38# - Go to oilshell.org repo and do:
39# ./deploy.sh site # copy release
40# ./deploy.sh bump-index-version
41# make
42# ./deploy.sh site # copy new index
43# ./deploy.sh bump-release-version
44# - Go to oilshell.org__deploy and "git add release/$VERSION".
45# - git commit -a
46
47set -o nounset
48set -o pipefail
49set -o errexit
50
51shopt -s strict:all 2>/dev/null || true # dogfood for OSH
52
53REPO_ROOT=$(cd $(dirname $0)/.. ; pwd)
54OIL_VERSION=$(head -n 1 oil-version.txt)
55
56source devtools/common.sh # banner
57source benchmarks/common.sh # BENCHMARK_DATA_OILS, OSH_CPP_BENCHMARK_DATA
58 # redefines OIL_VERSION as readonly
59
60readonly OSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/osh
61readonly YSH_RELEASE_BINARY=$REPO_ROOT/_tmp/oil-tar-test/oil-$OIL_VERSION/_bin/ysh
62
63log() {
64 echo "$@" 1>&2
65}
66
67make-release-branch() {
68 git checkout master
69 local name=release/$OIL_VERSION
70 git checkout -b $name
71 git push -u origin $name
72}
73
74ensure-smooth-build() {
75 # Stray files can mess up the unit tests
76 devtools/git.sh error-if-untracked
77
78 build/clean.sh
79
80 sudo -k; sudo true # clear and re-cache credentials
81
82 # Install with root privileges
83 _install
84}
85
86# For redoing a release. This is everything until you have to 'git pull' the
87# benchmark-data repo to make reports.
88#
89# PRECONDITION: $0 two-tarballs was run manually, which runs
90# ensure-smooth-build.
91auto-machine1() {
92 local resume=${1:-} # workaround for spec test flakiness bug
93 local resume2=${2:-} # skip past spec sanity check
94 local resume3=${3:-} # skip past metrics and wild tests
95 local resume4=${4:-} # skip past full spec tests
96
97 if test -z "$resume"; then
98 $0 build-and-test
99 fi
100
101 if test -z "$resume2"; then
102 _spec-sanity-check # just run a few spec tests
103 fi
104
105 if test -z "$resume3"; then
106 $0 metrics # this can catch bugs
107 test/wild.sh all
108 fi
109
110 if test -z "$resume4"; then
111 $0 spec-all # full spec test run
112 fi
113
114 $0 benchmark-run do_machine1
115}
116
117# Note: needs dep-benchmarks to run
118auto-machine2() {
119 ensure-smooth-build
120
121 # Note: this can't be done until we sync the oils-for-unix source from
122 # machine 1.
123 $0 benchmark-build
124 $0 benchmark-run
125}
126
127# TODO:
128# - enforce that there is a release/$VERSION branch?
129
130# oilshell.org__deploy/
131# releases.html
132# release/
133# $VERSION/
134# index.html # release page, from doc/release-index.md
135# oil-version.txt
136# release-date.txt
137# announcement.html # HTML redirect
138# changelog.html
139# doc/
140# index.html
141# ...
142# test/ # results
143# spec.wwz/
144# machine-lisa/
145# wild.wwz/
146# unit.wwz/
147# other.wwz/
148# gold.txt
149# parse-errors.txt
150# runtime-errors.txt
151# tools-deps.txt
152# osh-usage.txt
153# lossless.txt
154# tarball/ # log of building and running the tarball?
155# asan/ # spec tests or other?
156# # or it can be put under test/{spec,wild}
157# metrics.wwz/ # static metrics on source code?
158# line-counts/
159# nativedeps.txt (build/stats.sh line counts)
160# bytecode size, number of PyCodeObject
161# number of functions, classes, etc.?
162# bytecode/bundle size (binary size on x86_64 is in ovm-build.sh)
163# tarball size?
164# coverage.wwz/
165# unified/ # clang-coverage
166# benchmarks.wwz/
167# compute
168# osh-parser/
169# osh-runtime/
170# vm-baseline/
171# ...
172# startup/
173# download/ # What about native binaries?
174# 0.0.0/
175# oil-0.0.0.tar.xz
176
177_test-tarball() {
178 local name=${1:-hello}
179 local version=${2:-0.0.0}
180 local install=${3:-}
181
182 local tmp=_tmp/${name}-tar-test
183 rm -r -f $tmp
184 mkdir -p $tmp
185
186 pushd $tmp
187 tar --extract -z < ../../_release/$name-$version.tar.gz
188
189 cd $name-$version
190 ./configure
191
192 # Build the fast one for a test.
193 # TODO: Maybe edit the Makefile to change the top target.
194 local bin=_bin/${name}.ovm # not dbg
195 time make $bin
196 $bin --version
197
198 if test -n "$install"; then
199 sudo ./install
200 fi
201 popd
202}
203
204test-oil-tar() {
205 local install=${1:-} # non-empty to install
206 _test-tarball oil $(head -n 1 oil-version.txt) "$install"
207}
208
209_release-build() {
210 # NOTE: deps/from-tar.sh {configure,build}-python is assumed
211
212 # Build the oil tar
213 $0 oil
214
215 test-oil-tar
216
217 # For _spec-sanity-check
218 ln -s -f --no-target-directory -v oil.ovm $OSH_RELEASE_BINARY
219 ln -s -f --no-target-directory -v oil.ovm $YSH_RELEASE_BINARY
220}
221
222readonly HAVE_ROOT=1
223
224readonly -a MORE_TESTS=(
225 process-table
226 gold
227 ysh-ify
228 parse-errors runtime-errors
229 ysh-runtime-errors
230 ysh-parse-errors
231 ysh-every-string
232 lossless
233 osh-usage tools-deps
234 syscall
235)
236
237run-more-tests() {
238 for name in "${MORE_TESTS[@]}"; do
239 case $name in
240 gold)
241 if test -n "${OILS_HIJACK_SHEBANG:-}"; then
242 cat >&2 <<'EOF'
243=====
244WARNING: Skipping gold tests because $OILS_HIJACK_SHEBANG is set.'
245Run them manually with:
246
247 test/gold.sh run-for-release
248=====
249EOF
250 continue
251 fi
252 ;;
253 *)
254 banner "Test suite: $name"
255 ;;
256 esac
257
258 test/$name.sh run-for-release
259 done
260
261 ysh/run.sh run-for-release
262
263 data_lang/j8-errors.sh run-for-release
264}
265
266_spec-sanity-check() {
267 # Quick early test for _bin/osh and _bin/ysh
268
269 # TODO: Use --ovm-bin-dir
270 # Note: MAX_PROCS=1 prevents [#oil-dev > Random Spec Test Stoppages]
271 # Still need to fix that bug
272 MAX_PROCS=1 NUM_SPEC_TASKS=2 OSH_LIST="$OSH_RELEASE_BINARY" test/spec-py.sh osh-all
273 MAX_PROCS=1 NUM_SPEC_TASKS=2 YSH_LIST="$YSH_RELEASE_BINARY" test/spec-py.sh ysh-all
274}
275
276spec-all() {
277 ### Run all spec tests
278
279 test/stateful.sh soil-run # Same as CI
280
281 # Create the tests we're running
282 test/smoosh.sh make-spec
283
284 # TODO: Use --ovm-bin-dir
285 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
286 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
287 test/spec-py.sh all-and-smoosh
288
289 # Build $OSH_CPP_BENCHMARK_DATA
290 _build-oils-benchmark-data
291
292 # TODO: Use --oils-cpp-bin-dir
293 # Collect and publish stats about the C++ translation.
294 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh osh-all
295 YSH_CC="$YSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh ysh-all
296}
297
298spec-cpp() {
299 ### For repair
300
301 # TODO: Use --oils-cpp-bin-dir
302
303 # Quick
304 # NUM_SPEC_TASKS=2 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
305 OSH_CC="$OSH_CPP_BENCHMARK_DATA" test/spec-cpp.sh all
306}
307
308# For quickly debugging failures that don't happen in dev mode.
309spec-one() {
310 export OSH_LIST="$REPO_ROOT/bin/osh $OSH_RELEASE_BINARY"
311 export YSH_LIST="$REPO_ROOT/bin/ysh $YSH_RELEASE_BINARY"
312 test/spec.sh "$@"
313}
314
315build-and-test() {
316 ### Build tarballs and test them. And preliminaries like unit tests.
317
318 # TODO: Log this whole thing? Include logs with the /release/ page?
319
320 # Before doing anything
321 test/lint.sh soil-run
322
323 test/unit.sh run-for-release # Python unit tests
324
325 test/coverage.sh run-for-release # C++ unit tests
326
327 # App bundle
328 _release-build
329
330 # TODO: test oils-for-unix in Alpine chroot too.
331 # NOTE: Need test/alpine.sh download;extract;setup-dns,add-oil-build-deps,
332 # etc.
333 if test -n "$HAVE_ROOT"; then
334 # TODO: Factor out test/alpine.sh to test/chroot.sh
335 test/alpine.sh copy-tar '' oil
336 test/alpine.sh test-tar '' oil
337 fi
338
339 test/spec.sh smoke # Initial smoke test, slightly redundant.
340
341 run-more-tests
342}
343
344_install() {
345 test/spec-bin.sh install-shells-with-apt
346
347 # A subset of build/py.sh ubuntu-deps. (Do we need build-essential?)
348 #sudo apt install python-dev
349}
350
351_build-oils-benchmark-data() {
352 pushd $BENCHMARK_DATA_OILS
353 for variant in dbg opt; do
354 # DWARF version 4 is a hack for bloaty, which doesn't support version 5.
355 # I don't think this should affect benchmarks besides
356 # metrics/native-code.sh, so we don't bother building a separate binary.
357 # The Soil CI runs without this flag.
358 CXXFLAGS=-gdwarf-4 _build/oils.sh '' $variant SKIP_REBUILD
359 done
360 popd
361}
362
363benchmark-build() {
364 ### Build function on machine 2.
365
366 build/clean.sh
367 if test -n "$HAVE_ROOT"; then
368 _install
369 fi
370 build/py.sh all
371 _release-build
372}
373
374# Run benchmarks with the binary built out of the tarball.
375benchmark-run() {
376 local do_machine1=${1:-}
377
378 _build-oils-benchmark-data
379 OSH_OVM=$OSH_RELEASE_BINARY benchmarks/auto.sh all "$do_machine1"
380}
381
382_compressed-tarball() {
383 local name=${1:-hello}
384 local version=${2:-0.0.0}
385
386 local in=_release/$name.tar
387 local out=_release/$name-$version.tar.gz
388
389 # Overwrite it to cause rebuild of oil.tar
390 build/stamp.sh write-release-date
391
392 #make -d -r $in # To debug
393 make $in
394 time gzip -c $in > $out
395 ls -l $out
396
397 # xz version is considerably smaller. 1.15 MB vs. 1.59 MB.
398 local out2=_release/$name-$version.tar.xz
399 time xz -c $in > $out2
400 ls -l $out2
401}
402
403oil() {
404 _compressed-tarball oil $OIL_VERSION
405}
406
407hello() {
408 _compressed-tarball hello $(head -n 1 build/testdata/hello-version.txt)
409}
410
411
412_link() {
413 ln -s -f -v --no-target-directory "$@"
414}
415
416compress() {
417 local root=$PWD/_release/VERSION/
418
419 log '--- more-tests'
420 local out="$root/more-tests.wwz"
421 pushd _tmp
422 time zip -r -q $out suite-logs unit syscall process-table
423 popd
424
425 # This has HTML reports, .profraw files, and logs of stdout, e.g.
426 # mycpp-unit/gc_heap_test.log
427 # About 1.5 MB
428 log "--- coverage"
429 local out="$root/test/coverage.wwz"
430 pushd _test/clang-coverage
431 # This also saves the logs
432 time zip -r -q $out .
433 popd
434
435 log "--- test/spec"
436 local out="$root/test/spec.wwz"
437 pushd _tmp/spec
438 time zip -r -q $out . # recursive, quiet
439 popd
440
441 log "--- test/wild"
442 local out="$root/test/wild.wwz"
443 pushd _tmp/wild-www
444 time zip -r -q $out . # recursive, quiet
445 popd
446
447 # NOTE: must be /pub/metrics.wwz so that relative URLs like
448 # ../../../web/line-counts.css work. The Soil UI also relies on such
449 # relative URLs.
450 log "--- metrics"
451 local out="$root/pub/metrics.wwz"
452 pushd _tmp/metrics
453 time zip -r -q $out . # recursive, quiet
454 popd
455
456 # Ditto: pub/src-tree.wwz lines up with URLs in Soil
457 log "--- src-tree"
458 local out="$root/pub/src-tree.wwz"
459 pushd _tmp/src-tree-www
460 time zip -r -q $out . # recursive, quiet
461 popd
462
463 compress-benchmarks
464
465 tree _release/VERSION
466}
467
468compress-benchmarks() {
469 local root=$PWD/_release/VERSION/
470 mkdir -p $root
471
472 log "--- benchmarks"
473
474 local out="$root/benchmarks.wwz"
475
476 # - For benchmarks that run on multiple machines, technically we only need
477 # index.html, but include stage1 and stage2.
478 # - For those that run on single machines, we also archive the raw/ dir.
479 # - Although benchmarks/compute is saved in oilshell/benchmark-data
480 # - Note: _tmp/uftrace/{raw,stage1} are big (hundreds of MB), so leave them
481 # out
482
483 pushd _tmp
484 find \
485 osh-parser/{stage1,stage2,index.html} \
486 osh-runtime/{stage1,stage2,index.html} \
487 vm-baseline/{stage1,stage2,index.html} \
488 ovm-build/{stage1,stage2,index.html} \
489 compute/{raw,stage1,stage2,index.html} \
490 gc/{raw,stage2,index.html} \
491 gc-cachegrind/{raw,stage2,index.html} \
492 mycpp-examples/{raw,stage2,index.html} \
493 uftrace/{stage2,index.html} \
494 -type f \
495 | xargs --verbose -- zip -q $out
496 popd
497}
498
499line-counts() {
500 local out_dir=$1 # should be an absolute path
501 mkdir -p $out_dir
502
503 # Counting directly from the build.
504 metrics/tarball.sh linecount-pydeps > $out_dir/pydeps.txt
505 metrics/tarball.sh linecount-nativedeps > $out_dir/nativedeps.txt
506 metrics/tarball.sh linecount-oils-cpp > $out_dir/oils-cpp.txt
507
508 metrics/source-code.sh write-reports $out_dir # for-translation and overview
509 metrics/source-code.sh cloc-report > $out_dir/cloc-report.txt
510
511 # goes to _tmp/metrics/preprocessed
512 metrics/source-code.sh preprocessed
513}
514
515metrics() {
516 local out=_tmp/metrics
517 mkdir -p $out
518
519 line-counts $PWD/$out/line-counts
520
521 # For another .wwz file
522 doctools/src-tree.sh soil-run
523
524 metrics/bytecode.sh run-for-release
525 metrics/native-code.sh run-for-release
526 build/cpython-defs.sh run-for-release
527
528 tree $out
529}
530
531deploy-doc() {
532 local deploy_repo='../oilshell.org__deploy'
533 local release_root_dir="$deploy_repo/release"
534 local release_dir="$release_root_dir/$OIL_VERSION"
535
536 cp -v -r --force --no-target-directory \
537 _release/VERSION/ $release_dir/
538
539 # Generate release index.
540 html-index $release_root_dir _tmp/releases.html
541 cp -v _tmp/releases.html $deploy_repo
542
543 tree -L 3 $release_root_dir
544
545 ls -l $deploy_repo/releases.html
546}
547
548readonly DOWNLOAD_DIR='../oilshell.org__deploy/download/'
549
550# Generating releases.html requires the old tarballs!
551sync-old-tar() {
552 local user=$1 # required username
553 rsync --archive --verbose \
554 $user@oilshell.org:oilshell.org/download/ $DOWNLOAD_DIR
555}
556
557# I think these aren't checked into git? They can just be managed separately?
558# Or should you check in the sha checksums? Those will be in releases.html,
559# but a CSV might be nice.
560deploy-tar() {
561 mkdir -p $DOWNLOAD_DIR
562
563 cp -v \
564 _release/oil-$OIL_VERSION.tar.* _release/oils-for-unix-$OIL_VERSION.tar.* \
565 $DOWNLOAD_DIR
566
567 ls -l $DOWNLOAD_DIR
568}
569
570#
571# Generate releases.html.
572#
573
574# Examples of similar release HTML pages:
575# - https://golang.org/dl/ - "Older versions", sha1 / sha256.
576# - Python has all point releases in chronological order, and then a separate
577# page for each changelog. There is too much boilerplate maybe?
578# - It has release notes before the downloads. Not sure I like that.
579# - node.js: https://nodejs.org/en/
580# - user agent detection for the right binary -- meh I don't want that
581# - Ruby: https://www.ruby-lang.org/en/downloads/releases/
582# - https://www.lua.org/download.html
583
584# Columns: Date / Version / Docs / / Files
585# Changelog .xz
586# Install
587# Docs/
588#
589# The files could be a separate page and separate table? I could provide
590# pre-built versions eventually? Linux static versions?
591
592# TODO: Each of these would be a good candidate for a data frame! Data vs.
593# presentation.
594
595# Simple UI:
596# - home page shows latest version (source release for now, binary release later?)
597# - link to Changelog, INSTALL, doc index
598# - or see all releases
599# - Grey out older releases?
600
601# TODO: Should be sorted by date? How to do that, with bash array? Or Awk?
602# $timestamp $version $timestamp file? And then sort -n I guess? Change
603# the release date format. It will use Unix timestamp (OK until 2038!)
604
605_html-index() {
606 local release_root_dir=$1 # the directory we want to make an index of
607
608 for entry in $release_root_dir/*; do
609 if ! test -d $entry; then
610 continue
611 fi
612 local dir=$entry
613
614 local version
615 version=$(head -n 1 $dir/oil-version.txt)
616 local release_date
617 release_date=$(head -n 1 $dir/release-date.txt)
618
619 log "-- $dir"
620 log "Version: $version"
621 log "Release Date: $release_date"
622 log ""
623
624 echo "$release_date $version"
625 done > _tmp/release-meta.txt
626
627 # Reverse sort by release date
628 sort -r _tmp/release-meta.txt > _tmp/sorted-releases.txt
629
630 while read date _ version; do
631 log "Release Date: $date"
632 log "Version: $version"
633
634 # anchor
635 cat <<EOF
636<tr>
637 <td>
638 <span class="date">$date</span>
639 </td>
640 <td>
641 <a name="$version"></a>
642 <span class="version-number">$version</span>
643 </td>
644 <td>
645 <p> <a href="release/$version/announcement.html">Announcement</a>
646 &nbsp; | &nbsp; <a href="release/$version/">Docs and Details</a>
647 </p>
648 </td>
649</tr>
650EOF
651
652 build/doc.sh tarball-links-row-html $version
653
654 cat <<EOF
655<tr>
656 <td colspan="3">
657 <div style="padding: 1em;" >
658 </div>
659 </td>
660</tr>
661
662EOF
663
664 done < _tmp/sorted-releases.txt
665}
666
667_releases-html-header() {
668 # TODO: use html-head here, and publish web/*.css somewhere outside of
669 # /release/$VERSION/? The list of all releases isn't versioned for obvious
670 # reasons. Other docs are in the oilshell.org repo using the all-2020.css
671 # bundle.
672
673 cat <<EOF
674<!DOCTYPE html>
675<html>
676 <head>
677 <meta name="viewport" content="width=device-width, initial-scale=1">
678 <title>Oils Releases</title>
679 <style>
680EOF
681
682 cat web/base.css
683 cat web/release-index.css
684
685cat <<EOF
686 h1 {
687 text-align: center;
688 }
689 </style>
690 </head>
691 <body class="width50">
692 <p id="home-link">
693 <a href="/">oilshell.org</a>
694 </p>
695 <h1>Oils Releases</h1>
696
697 <table class="release-table">
698EOF
699}
700
701html-index() {
702 local release_root_dir=$1
703 local out=${2:-_tmp/releases.html}
704
705 { _releases-html-header
706 _html-index $release_root_dir
707
708 cat <<EOF
709 </table>
710 </body>
711</html>
712EOF
713
714 } > $out
715
716 ls -l $out
717}
718
719# For quickly iterating on tarball size reductions.
720tarball-size() {
721 make clean-repo
722 make _bin/oil.ovm-dbg # faster way to build bytecode
723 oil # make tarball
724 test-oil-tar # Ctrl-C this, then run metrics/tarball.sh
725}
726
727dep-smoosh() {
728 local repo=~/git/languages/smoosh
729 if ! test -d $repo; then
730 local base_dir=$(dirname $repo)
731 mkdir -p $base_dir
732 pushd $base_dir
733 git clone git@github.com:mgree/smoosh.git
734 popd
735 fi
736}
737
738dep-benchmarks() {
739 ### Before auto-machine2
740
741 # 2023-07: Also need deps/from-tar.sh {configure,build}-cpython
742
743 benchmarks/osh-runtime.sh download
744 benchmarks/osh-runtime.sh extract
745
746 benchmarks/ovm-build.sh download
747 benchmarks/ovm-build.sh extract-other
748
749 # For ovm-build benchmark.
750 deps/from-binary.sh download-clang
751 deps/from-binary.sh extract-clang
752}
753
754more-release-deps() {
755 # List of deps that are NOT in soil/worker.sh here
756 # https://github.com/oilshell/oil/issues/926
757
758 # TODO: Make a container image for these.
759 if false; then
760 # TODO: Did this manually
761 # test/alpine.sh
762 # dep-alpine
763
764 # test/smoosh.sh
765 dep-smoosh
766
767 dep-benchmarks
768 fi
769}
770
771py-tarball() {
772 local in=_release/oil.tar
773 local out=_release/oil-$OIL_VERSION.tar.gz
774
775 make $in
776 time gzip -c $in > $out
777 ls -l $out
778
779 test-oil-tar
780}
781
782native-tarball() {
783 # oils-for-unix
784 devtools/release-native.sh make-tar
785 # Also install as root
786 devtools/release-native.sh extract-for-benchmarks INSTALL
787}
788
789two-tarballs() {
790 ### First step of release. Assume that CI passes
791
792 ensure-smooth-build
793
794 build/py.sh all
795 # "Base state" for repo scripts
796 ./NINJA-config.sh
797
798 py-tarball
799
800 native-tarball
801}
802
803upload-tmp() {
804 local tarball=$1
805 local user=$2
806
807 scp $tarball $user@oilshell.org:tmp/
808}
809
810sync-tmp() {
811 local user=$1
812 local dest=${2:-_tmp/candidates}
813 mkdir -p $dest
814 rsync --archive --verbose $user@oilshell.org:tmp/ $dest
815}
816
817"$@"