OILS / build / deps.sh View on Github | oilshell.org

1199 lines, 623 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
13#
14# TODO: Do we need something faster, just python2, re2c, and cmark?
15#
16# - build/deps.sh fetch-py
17# - build/deps.sh install-wedges-py
18#
19# TODO: Can we make most of them non-root deps? This requires rebuilding
20# containers, which requires podman.
21#
22# rm -r -f ~/wedge # would be better
23
24set -o nounset
25set -o pipefail
26set -o errexit
27
28REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
29
30source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
31source deps/from-apt.sh # PY3_BUILD_DEPS
32#source deps/podman.sh
33source devtools/run-task.sh # run-task
34source test/tsv-lib.sh # tsv-concat
35source web/table/html.sh # table-sort-{begin,end}
36
37# Also in build/dev-shell.sh
38USER_WEDGE_DIR=~/wedge/oils-for-unix.org
39ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
40
41readonly DEPS_SOURCE_DIR=_build/deps-source
42
43readonly RE2C_VERSION=3.0
44readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
45
46readonly CMARK_VERSION=0.29.0
47readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
48
49readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
50
51readonly PY2_VERSION=2.7.18
52readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
53
54readonly PY3_VERSION=3.10.4
55readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
56
57readonly BASH_VER=4.4 # don't clobber BASH_VERSION
58readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
59
60# Another version of bash to test
61readonly BASH5_VER=5.2
62readonly BASH5_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH5_VER.tar.gz"
63
64readonly DASH_VERSION=0.5.10.2
65readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
66
67readonly ZSH_VERSION=5.1.1
68readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
69
70readonly MKSH_VERSION=R52c
71readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
72
73readonly BUSYBOX_VERSION='1.35.0'
74readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
75
76readonly YASH_VERSION=2.49
77readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
78
79readonly MYPY_GIT_URL=https://github.com/python/mypy
80readonly MYPY_VERSION=0.780
81
82readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
83
84# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
85# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
86readonly PYFLAKES_VERSION=2.4.0
87#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
88# 2023-07: Mirrored to avoid network problem on broome during release
89readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
90
91readonly BLOATY_VERSION=1.1
92readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
93
94readonly UFTRACE_VERSION=0.13
95readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
96
97readonly SOUFFLE_VERSION=2.4.1
98readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
99
100log() {
101 echo "$0: $@" >& 2
102}
103
104die() {
105 log "$@"
106 exit 1
107}
108
109rm-oils-crap() {
110 ### When you want to start over
111
112 rm -r -f -v ~/wedge
113 sudo rm -r -f -v /wedge
114}
115
116# Note: git is an implicit dependency -- that's how we got the repo in the
117# first place!
118
119# python2-dev is no longer available on Debian 12
120# python-dev also seems gone
121#
122# wget: for fetching wedges (not on Debian by default!)
123# tree: tiny package that's useful for showing what we installed
124# g++: essential
125# libreadline-dev: needed for the build/prepare.sh Python build.
126# gawk: used by spec-runner.sh for the special match() function.
127# cmake: for cmark
128# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
129# as well
130readonly -a WEDGE_DEPS_DEBIAN=(
131 bzip2
132 wget
133 tree
134 gawk
135 g++
136 ninja-build
137 cmake
138 libreadline-dev
139 systemtap-sdt-dev
140
141 # for Souffle, flex and bison
142 #flex bison
143
144 "${PY3_BUILD_DEPS[@]}"
145)
146
147readonly -a WEDGE_DEPS_ALPINE=(
148 bzip2
149 xz
150
151 wget tree gawk
152
153 gcc g++
154 ninja-build
155 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
156 ninja-is-really-ninja
157 cmake
158
159 readline-dev
160 zlib-dev
161 libffi-dev
162 openssl-dev
163
164 ncurses-dev
165
166 # for Souffle, flex and bison
167 #flex bison
168)
169
170readonly -a WEDGE_DEPS_FEDORA=(
171
172 # Weird, Fedora doesn't have these by default!
173 hostname
174 tar
175 bzip2
176
177 # https://packages.fedoraproject.org/pkgs/wget/wget/
178 wget
179 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
180 tree
181 gawk
182
183 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
184 gcc gcc-c++
185
186 ninja-build
187 cmake
188
189 readline-devel
190
191 # Like PY3_BUILD_DEPS
192 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
193 zlib-devel
194 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
195 libffi-devel
196 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
197 openssl-devel
198
199 # For building zsh from source?
200 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
201 ncurses-devel
202 #libcap-devel
203
204 # still have a job control error compiling bash
205 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
206 # glibc-devel
207)
208
209install-debian-packages() {
210 ### Packages for build/py.sh all, building wedges, etc.
211
212 set -x # show what needs sudo
213
214 # pass -y for say gitpod
215 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
216 set +x
217
218 # maybe pass -y through
219 test/spec-bin.sh install-shells-with-apt "$@"
220}
221
222install-ubuntu-packages() {
223 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
224 install-debian-packages "$@"
225}
226
227wedge-deps-debian() {
228 # Install packages without prompt
229
230 # 2024-02 - there was an Ubuntu update, and we started needing this
231 sudo apt-get -y update
232
233 install-debian-packages -y
234}
235
236wedge-deps-fedora() {
237 # https://linuxconfig.org/install-development-tools-on-redhat-8
238 # Trying to get past compile errors
239 # sudo dnf group install --assumeyes 'Development Tools'
240
241 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
242}
243
244wedge-deps-alpine() {
245 # https://linuxconfig.org/install-development-tools-on-redhat-8
246 # Trying to get past compile errors
247 # sudo dnf group install --assumeyes 'Development Tools'
248
249 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
250}
251
252#
253# Unused patch, was experiment for Fedora
254#
255
256get-typed-ast-patch() {
257 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
258}
259
260# Work around typed_ast bug:
261# https://github.com/python/typed_ast/issues/169
262#
263# Apply this patch
264# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
265#
266# typed_ast is tarred up though
267patch-typed-ast() {
268 local package_dir=_cache/py3-libs
269 local patch=$PWD/deps/typed_ast.patch
270
271 pushd $package_dir
272 cat $patch
273 echo
274
275 local dir=typed_ast-1.4.3
276 local tar=typed_ast-1.4.3.tar.gz
277
278 echo OLD
279 ls -l $tar
280 echo
281
282 rm -r -f -v $dir
283 tar -x -z < $tar
284
285 pushd $dir
286 patch -p1 < $patch
287 popd
288 #find $dir
289
290 # Create a new one
291 tar --create --gzip --file $tar typed_ast-1.4.3
292
293 echo NEW
294 ls -l $tar
295 echo
296
297 popd
298}
299
300#
301# Fetch
302#
303
304download-to() {
305 local dir=$1
306 local url=$2
307 wget --no-clobber --directory-prefix "$dir" "$url"
308}
309
310maybe-extract() {
311 local wedge_dir=$1
312 local tar_name=$2
313 local out_dir=$3
314
315 if test -d "$wedge_dir/$out_dir"; then
316 log "Not extracting because $wedge_dir/$out_dir exists"
317 return
318 fi
319
320 local tar=$wedge_dir/$tar_name
321 case $tar_name in
322 *.gz|*.tgz) # mksh ends with .tgz
323 flag='--gzip'
324 ;;
325 *.bz2)
326 flag='--bzip2'
327 ;;
328 *.xz)
329 flag='--xz'
330 ;;
331 *)
332 die "tar with unknown extension: $tar_name"
333 ;;
334 esac
335
336 tar --extract $flag --file $tar --directory $wedge_dir
337}
338
339clone-mypy() {
340 ### replaces deps/from-git
341 local dest_dir=$1
342 local version=${2:-$MYPY_VERSION}
343
344 local dest=$dest_dir/mypy-$version
345 if test -d $dest; then
346 log "Not cloning because $dest exists"
347 return
348 fi
349
350 # v$VERSION is a tag, not a branch
351
352 # size optimization: --depth=1 --shallow-submodules
353 # https://git-scm.com/docs/git-clone
354
355 git clone --recursive --branch v$version \
356 --depth=1 --shallow-submodules \
357 $MYPY_GIT_URL $dest
358
359 # TODO: verify commit checksum
360}
361
362copy-source-medo() {
363 mkdir -p $DEPS_SOURCE_DIR
364
365 # Copy the whole tree, including the .treeptr files
366 cp --verbose --recursive --no-target-directory \
367 deps/source.medo/ $DEPS_SOURCE_DIR/
368}
369
370fetch-spec-bin() {
371 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
372 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" bash-$BASH_VER
373
374 download-to $DEPS_SOURCE_DIR/bash "$BASH5_URL"
375 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH5_URL)" bash-$BASH5_VER
376
377 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
378 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
379
380 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
381 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
382
383 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
384 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
385
386 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
387 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
388
389 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
390 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$YASH_VERSION
391
392 # Patch: this tarball doesn't follow the convention $name-$version
393 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
394 pushd $DEPS_SOURCE_DIR/mksh
395 mv -v mksh mksh-$MKSH_VERSION
396 popd
397 fi
398}
399
400fetch() {
401 local py_only=${1:-}
402
403 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
404 # would do: fetch compressed tarballs designated by .treeptr files, and
405 # expand them.
406
407 # _build/deps-source/
408 # re2c/
409 # WEDGE
410 # re2c-3.0/ # expanded .tar.xz file
411
412 copy-source-medo
413
414 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
415 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
416 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
417 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
418
419 if test -n "$py_only"; then
420 log "Fetched dependencies for 'build/py.sh'"
421 return
422 fi
423
424 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
425 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
426 pyflakes-$PYFLAKES_VERSION
427
428 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
429 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
430 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
431 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
432
433 fetch-spec-bin
434
435 # bloaty and uftrace are for benchmarks, in containers
436 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
437 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
438 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" bloaty-$BLOATY_VERSION
439 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" uftrace-$UFTRACE_VERSION
440
441 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
442
443 # It's also copied into a wedge in install-wedges.
444 clone-mypy $DEPS_SOURCE_DIR/mypy
445
446 if false; then
447 download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
448 maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
449 fi
450
451 if command -v tree > /dev/null; then
452 tree -L 2 $DEPS_SOURCE_DIR
453 fi
454}
455
456fetch-py() {
457 fetch py_only
458}
459
460mirror-pyflakes() {
461 ### Workaround for network error during release
462 scp \
463 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
464 oilshell.org:oilshell.org/blob/
465}
466
467wedge-exists() {
468 ### Does an installed wedge already exist?
469
470 local name=$1
471 local version=$2
472 local wedge_dir=${3:-/wedge/oils-for-unix.org}
473
474 local installed=$wedge_dir/pkg/$name/$version
475
476 if test -d $installed; then
477 log "$installed already exists"
478 return 0
479 else
480 return 1
481 fi
482}
483
484#
485# Install
486#
487
488# TODO: py3-libs needs to be a WEDGE, so that that you can run
489# 'wedge build deps/source.medo/py3-libs/' and then get it in
490#
491# _build/wedge/{absolute,relative} # which one?
492#
493# It needs a BUILD DEPENDENCY on:
494# - the python3 wedge, so you can do python3 -m pip install.
495# - the mypy repo, which has test-requirements.txt
496
497download-py3-libs() {
498 ### Download source/binary packages, AFTER python3 is installed
499
500 # Note that this is NOT source code; there is binary code, e.g. in
501 # lxml-*.whl
502
503 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
504 local py_package_dir=_cache/py3-libs
505 mkdir -p $py_package_dir
506
507 # Avoids a warning, but doesn't fix typed_ast
508 #python3 -m pip download -d $py_package_dir wheel
509
510 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
511 python3 -m pip download -d $py_package_dir pexpect
512}
513
514install-py3-libs-in-venv() {
515 local venv_dir=$1
516 local mypy_dir=$2 # This is a param for host build vs. container build
517 local package_dir=_cache/py3-libs
518
519 source $venv_dir/bin/activate # enter virtualenv
520
521 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
522 # conflicts with MyPy deps!
523 # "ERROR: pip's dependency resolver does not currently take into account all
524 # the packages that are installed."
525
526 # --find-links uses a "cache dir" for packages (weird flag name!)
527
528 # Avoids a warning, but doesn't fix typed_ast
529 #time python3 -m pip install --find-links $package_dir wheel
530
531 # for mycpp/
532 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
533
534 # pexpect: for spec/stateful/*.py
535 time python3 -m pip install --find-links $package_dir pexpect
536}
537
538install-py3-libs-from-cache() {
539
540 # As well as end users
541
542 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
543
544 local py3
545 py3=$(command -v python3)
546 case $py3 in
547 *wedge/oils-for-unix.org/*)
548 ;;
549 *)
550 die "python3 is '$py3', but expected it to be in a wedge"
551 ;;
552 esac
553
554 log "Ensuring pip is installed (interpreter $(command -v python3)"
555 python3 -m ensurepip
556
557 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
558 log "Creating venv in $venv_dir"
559
560 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
561 # the /wedge we just built
562 python3 -m venv $venv_dir
563
564 log "Installing MyPy deps in venv"
565
566 # Run in a subshell because it mutates shell state
567 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
568}
569
570install-py3-libs() {
571 ### Invoked by Dockerfile.cpp-small, etc.
572
573 download-py3-libs
574 install-py3-libs-from-cache
575}
576
577# OBSOLETE in favor of install-spec-bin-fast
578install-spec-bin() {
579 if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
580 deps/wedge.sh unboxed _build/deps-source/dash
581 fi
582
583 if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
584 deps/wedge.sh unboxed _build/deps-source/mksh
585 fi
586
587 if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
588 deps/wedge.sh unboxed _build/deps-source/busybox
589 fi
590
591 # Fedora compile error - count_all_jobs
592 if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
593 deps/wedge.sh unboxed _build/deps-source/bash
594 fi
595
596 # Fedora compiler error
597 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
598 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
599 # like ncurses-devel should fix this, but it doesn't
600 #
601 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
602 #
603 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
604 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
605 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
606 #
607 # I think the ./configure is out of sync with the actual build?
608
609 if ! wedge-exists zsh $ZSH_VERSION ''; then
610 deps/wedge.sh unboxed _build/deps-source/zsh
611 fi
612
613 return
614
615 # Hm this has problem with out-of-tree build? I think Oils does too actually
616 if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
617 deps/wedge.sh unboxed _build/deps-source/yash
618 fi
619}
620
621# TODO:
622# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
623# already in each WEDGE file
624
625py-wedges() {
626 ### for build/py.sh all
627
628 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
629 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
630 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
631 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
632}
633
634cpp-wedges() {
635 ### for ninja / mycpp translation
636
637 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
638 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
639
640 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
641 # separately for now
642 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
643}
644
645spec-bin-wedges() {
646 ### for test/spec-py.sh osh-all
647
648 echo dash $DASH_VERSION $USER_WEDGE_DIR
649 echo bash $BASH_VER $USER_WEDGE_DIR
650 echo bash $BASH5_VER $USER_WEDGE_DIR
651 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
652 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
653 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
654 echo yash $YASH_VERSION $USER_WEDGE_DIR
655}
656
657contributor-wedges() {
658 py-wedges
659 cpp-wedges
660 spec-bin-wedges
661}
662
663extra-wedges() {
664 # Contributors don't need uftrace, bloaty, and probably R-libs
665 # Although R-libs could be useful for benchmarks
666
667 # Test both outside the contianer, as well as inside?
668 echo uftrace $UFTRACE_VERSION $ROOT_WEDGE_DIR
669
670 #echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
671}
672
673timestamp() {
674 date '+%H:%M:%S'
675}
676
677my-time-tsv() {
678 python3 benchmarks/time_.py \
679 --tsv \
680 --time-span --rusage \
681 "$@"
682}
683
684maybe-install-wedge() {
685 local name=$1
686 local version=$2
687 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
688
689 local task_file=$WEDGE_LOG_DIR/$name-$version.task.tsv
690 local log_file=$WEDGE_LOG_DIR/$name-$version.log.txt
691
692 echo " TASK $(timestamp) $name $version > $log_file"
693
694 # python3 because it's OUTSIDE the container
695 # Separate columns that could be joined: number of files, total size
696 my-time-tsv --print-header \
697 --field xargs_slot \
698 --field wedge \
699 --field wedge_HREF \
700 --field version \
701 --output $task_file
702
703 if wedge-exists "$name" "$version" "$wedge_dir"; then
704 echo "CACHED $(timestamp) $name $version"
705 return
706 fi
707
708 local -a cmd=( deps/wedge.sh unboxed _build/deps-source/$name/ $version)
709
710 set +o errexit
711 my-time-tsv \
712 --field "$XARGS_SLOT" \
713 --field "$name" \
714 --field "$name-$version.log.txt" \
715 --field "$version" \
716 --append \
717 --output $task_file \
718 "${cmd[@]}" "$@" >$log_file 2>&1
719 local status=$?
720 set -o errexit
721
722 if test "$status" -eq 0; then
723 echo " OK $(timestamp) $name $version"
724 else
725 echo " FAIL $(timestamp) $name $version"
726 fi
727}
728
729dummy-task() {
730 ### For testing log capture
731 local name=$1
732 local version=$2
733
734 echo "Building $name $version"
735
736 # random float between 0 and 3
737 # weirdly we need a seed from bash
738 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
739 local secs
740 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
741
742 echo "sleep $secs"
743 sleep $secs
744
745 echo 'stdout'
746 log 'stderr'
747
748 if test $name = 'mksh'; then
749 echo "simulate failure for $name"
750 exit 2
751 fi
752}
753
754readonly WEDGE_LOG_DIR=_build/wedge/logs
755
756dummy-task-wrapper() {
757 # Similar to test/common.sh run-task-with-status, used by
758 # test/{spec,wild}-runner.sh
759 local name=$1
760 local version=$2
761
762 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
763 local log_file=$WEDGE_LOG_DIR/$name.log.txt
764
765 echo " TASK $(timestamp) $name $version > $log_file"
766
767 # python3 because it's OUTSIDE the container
768 # Separate columns that could be joined: number of files, total size
769 my-time-tsv --print-header \
770 --field xargs_slot \
771 --field wedge \
772 --field wedge_HREF \
773 --field version \
774 --output $task_file
775
776 my-time-tsv \
777 --field "$XARGS_SLOT" \
778 --field "$name" \
779 --field "$name.log.txt" \
780 --field "$version" \
781 --append \
782 --output $task_file \
783 $0 dummy-task "$@" >$log_file 2>&1 || true
784
785 echo " DONE $(timestamp) $name $version"
786}
787
788html-head() {
789 # python3 because we're outside containers
790 PYTHONPATH=. python3 doctools/html_head.py "$@"
791}
792
793index-html() {
794 local tasks_tsv=$1
795
796 local base_url='../../../web'
797 html-head --title 'Wedge Builds' \
798 "$base_url/ajax.js" \
799 "$base_url/table/table-sort.js" \
800 "$base_url/table/table-sort.css" \
801 "$base_url/base.css"\
802
803 table-sort-begin 'width60'
804
805 cat <<EOF
806 <p id="home-link">
807 <a href="/">oilshell.org</a>
808 </p>
809
810 <h1>Wedge Builds</h1>
811EOF
812
813 tsv2html3 $tasks_tsv
814
815 cat <<EOF
816 <p>
817 <a href="tasks.tsv">tasks.tsv</a>
818 </p>
819EOF
820
821 table-sort-end 'tasks' # ID for sorting
822}
823
824NPROC=$(nproc)
825#NPROC=1
826
827install-wedge-list() {
828 ### Reads task rows from stdin
829 local parallel=${1:-}
830
831 mkdir -p _build/wedge/logs
832
833 local -a flags
834 if test -n "$parallel"; then
835 log ""
836 log "=== Installing wedges with $NPROC jobs in parallel"
837 log ""
838 flags=( -P $NPROC )
839 else
840 log ""
841 log "=== Installing wedges serially"
842 log ""
843 fi
844
845 # Reads from stdin
846 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
847 #
848 # $name $version $wedge_dir
849 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
850
851 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
852}
853
854write-task-report() {
855 local tasks_tsv=_build/wedge/logs/tasks.tsv
856
857 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
858 log "Wrote $tasks_tsv"
859
860 # TODO: version can be right-justified?
861 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
862column_name type precision strftime
863status integer 0 -
864elapsed_secs float 1 -
865user_secs float 1 -
866start_time float 1 %H:%M:%S
867end_time float 1 %H:%M:%S
868sys_secs float 1 -
869max_rss_KiB integer 0 -
870xargs_slot integer 0 -
871wedge string 0 -
872wedge_HREF string 0 -
873version string 0 -
874EOF
875
876 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
877 log "Wrote $WEDGE_LOG_DIR/index.html"
878}
879
880install-spec-bin-fast() {
881 spec-bin-wedges | install-wedge-list T
882 write-task-report
883}
884
885fake-py3-libs-wedge() {
886 local name=py3-libs
887 local version=$PY3_LIBS_VERSION
888
889 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
890 local log_file=$WEDGE_LOG_DIR/$name.log.txt
891
892 my-time-tsv --print-header \
893 --field xargs_slot \
894 --field wedge \
895 --field wedge_HREF \
896 --field version \
897 --output $task_file
898
899 # There is no xargs slot!
900 my-time-tsv \
901 --field "-1" \
902 --field "$name" \
903 --field "$name.log.txt" \
904 --field "$version" \
905 --append \
906 --output $task_file \
907 $0 install-py3-libs >$log_file 2>&1 || true
908
909 echo " FAKE $(timestamp) $name $version"
910}
911
912install-wedges-fast() {
913 local extra=${1:-}
914
915 echo " START $(timestamp)"
916
917 # Do all of them in parallel
918 if test -n "$extra"; then
919 { contributor-wedges; extra-wedges; } | install-wedge-list T
920 else
921 contributor-wedges | install-wedge-list T
922 fi
923
924 fake-py3-libs-wedge
925 echo " END $(timestamp)"
926
927 write-task-report
928}
929
930install-wedges-soil() {
931 install-wedges-fast extra
932}
933
934# OBSOLETE in favor of install-wedges-fast
935install-wedges() {
936 local py_only=${1:-}
937
938 # TODO:
939 # - Make all of these RELATIVE wedges
940 # - Add
941 # - unboxed-rel-smoke-test -- move it inside container
942 # - rel-smoke-test -- mount it in a different location
943
944 if ! wedge-exists cmark $CMARK_VERSION; then
945 deps/wedge.sh unboxed _build/deps-source/cmark/
946 fi
947
948 if ! wedge-exists re2c $RE2C_VERSION; then
949 deps/wedge.sh unboxed _build/deps-source/re2c/
950 fi
951
952 if ! wedge-exists python2 $PY2_VERSION; then
953 deps/wedge.sh unboxed _build/deps-source/python2/
954 fi
955
956 if test -n "$py_only"; then
957 log "Installed dependencies for 'build/py.sh'"
958 return
959 fi
960
961 # Just copy this source tarball
962 if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
963 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
964 mkdir -p $dest_dir
965
966 cp --verbose --recursive --no-target-directory \
967 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
968 fi
969
970 if ! wedge-exists python3 $PY3_VERSION; then
971 deps/wedge.sh unboxed _build/deps-source/python3/
972 fi
973
974 # Copy all the contents, except for .git folder.
975 if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
976
977 # NOTE: We have to also copy the .git dir, because it has
978 # .git/modules/typeshed
979 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
980 mkdir -p $dest_dir
981
982 # Note: pack files in .git/modules/typeshed/objects/pack are read-only
983 # this can fail
984 cp --verbose --recursive --no-target-directory \
985 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
986 fi
987
988 if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
989 download-py3-libs
990 # This patch doesn't work?
991 # patch-typed-ast
992 install-py3-libs
993 fi
994
995 if ! wedge-exists souffle $SOUFFLE_VERSION $USER_WEDGE_DIR; then
996 deps/wedge.sh unboxed _build/deps-source/souffle/
997 fi
998
999 if command -v tree > /dev/null; then
1000 tree -L 3 $USER_WEDGE_DIR
1001 echo
1002 tree -L 3 /wedge/oils-for-unix.org
1003 fi
1004}
1005
1006install-wedges-py() {
1007 install-wedges py_only
1008}
1009
1010#
1011# Unboxed wedge builds
1012#
1013
1014uftrace-host() {
1015 ### built on demand; run $0 first
1016
1017 # BUG: doesn't detect python3
1018 # WEDGE tells me that it depends on pkg-config
1019 # 'apt-get install pkgconf' gets it
1020 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
1021
1022 deps/wedge.sh unboxed _build/deps-source/uftrace
1023}
1024
1025bloaty-host() {
1026 deps/wedge.sh unboxed _build/deps-source/bloaty
1027}
1028
1029R-libs-host() {
1030 deps/wedge.sh unboxed _build/deps-source/R-libs
1031}
1032
1033#
1034# Wedges built inside a container, for copying into a container
1035#
1036
1037boxed-wedges() {
1038 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
1039
1040 #export-podman
1041
1042 # TODO:
1043 #
1044 # - Add equivalents of spec-bin
1045 # - Use the same manifest as install-wedges-fast
1046 # - so then you can delete the _build/wedge dir to re-run it
1047 # - use xargs -n 1 so it's done serially
1048
1049 # - Do these lazily like we do in install-wedges-fast?
1050
1051 # We can test if the dir _build/wedge/binary/oils-for-unix.org/pkg/FOO exists
1052 # if wedge-exists "$name" "$version" "$wedge_dir"; then
1053 # echo "CACHED $(timestamp) $name $version"
1054 # return
1055 # fi
1056
1057 if false; then
1058 deps/wedge.sh boxed deps/source.medo/time-helper
1059 deps/wedge.sh boxed deps/source.medo/cmark/
1060 deps/wedge.sh boxed deps/source.medo/re2c/
1061 deps/wedge.sh boxed deps/source.medo/python3/
1062 fi
1063
1064 if false; then
1065 deps/wedge.sh boxed deps/source.medo/bloaty/
1066 fi
1067
1068 if true; then
1069 # build with debian-12, because soil-benchmarks2 is, because it has R
1070 deps/wedge.sh boxed deps/source.medo/uftrace/ '' debian-12
1071 # python2 needed everywhere
1072 #deps/wedge.sh boxed deps/source.medo/python2/ '' debian-12
1073
1074 # TODO: build with debian-12
1075 # Used in {benchmarks,benchmarks2,other-tests}
1076 #deps/wedge.sh boxed deps/source.medo/R-libs/ '' debian-12
1077 fi
1078}
1079
1080boxed-spec-bin() {
1081 if false; then
1082 deps/wedge.sh boxed deps/source.medo/bash '4.4'
1083 deps/wedge.sh boxed deps/source.medo/bash '5.2'
1084
1085 deps/wedge.sh boxed deps/source.medo/dash
1086 deps/wedge.sh boxed deps/source.medo/mksh
1087 fi
1088
1089 if true; then
1090 # Note: zsh requires libncursesw5-dev
1091 #deps/wedge.sh boxed deps/source.medo/zsh
1092
1093 #deps/wedge.sh boxed deps/source.medo/busybox
1094
1095 # Problem with out of tree build, as above. Skipping for now
1096 deps/wedge.sh boxed deps/source.medo/yash
1097 echo
1098 fi
1099}
1100
1101#
1102# Report
1103#
1104
1105commas() {
1106 # Wow I didn't know this :a trick
1107 #
1108 # OK this is a label and a loop, which makes sense. You can't do it with
1109 # pure regex.
1110 #
1111 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1112 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1113 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1114}
1115
1116wedge-sizes() {
1117 local tmp=_tmp/wedge-sizes.txt
1118
1119 # -b is --bytes, but use short flag for busybox compat
1120 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1121 { print $0 # print the line
1122 total_bytes += $1 # accumulate
1123 }
1124END { print total_bytes " TOTAL" }
1125' > $tmp
1126
1127 # printf justifies du output
1128 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1129 echo
1130
1131 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1132 #echo
1133}
1134
1135wedge-report() {
1136 # 4 levels deep shows the package
1137 if command -v tree > /dev/null; then
1138 tree -L 4 /wedge ~/wedge
1139 echo
1140 fi
1141
1142 wedge-sizes
1143
1144 local tmp=_tmp/wedge-manifest.txt
1145
1146 echo 'Biggest files'
1147 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1148 # busybox find doesn't have -printf
1149 echo 'find -printf failed'
1150 return
1151 fi
1152
1153 set +o errexit # ignore SIGPIPE
1154 sort -n --reverse $tmp | head -n 20 | commas
1155 set -o errexit
1156
1157 echo
1158
1159 # Show the most common file extensions
1160 #
1161 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1162 # most common
1163 #
1164 # There are also duplicate .a files for Python -- should look at how distros
1165 # get rid of those
1166
1167 cat $tmp | python3 -c '
1168import os, sys, collections
1169
1170bytes = collections.Counter()
1171files = collections.Counter()
1172
1173for line in sys.stdin:
1174 size, path = line.split(None, 1)
1175 path = path.strip() # remove newline
1176 _, ext = os.path.splitext(path)
1177 size = int(size)
1178
1179 bytes[ext] += size
1180 files[ext] += 1
1181
1182#print(bytes)
1183#print(files)
1184
1185n = 20
1186
1187print("Most common file types")
1188for ext, count in files.most_common()[:n]:
1189 print("%10d %s" % (count, ext))
1190
1191print()
1192
1193print("Total bytes by file type")
1194for ext, total_bytes in bytes.most_common()[:n]:
1195 print("%10d %s" % (total_bytes, ext))
1196' | commas
1197}
1198
1199run-task "$@"