OILS / build / deps.sh View on Github | oilshell.org

1098 lines, 577 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
13#
14# TODO: Do we need something faster, just python2, re2c, and cmark?
15#
16# - build/deps.sh fetch-py
17# - build/deps.sh install-wedges-py
18#
19# TODO: Can we make most of them non-root deps? This requires rebuilding
20# containers, which requires podman.
21#
22# rm -r -f ~/wedge # would be better
23
24set -o nounset
25set -o pipefail
26set -o errexit
27
28REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
29
30source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
31source deps/from-apt.sh # PY3_BUILD_DEPS
32#source deps/podman.sh
33source devtools/run-task.sh # run-task
34source test/tsv-lib.sh # tsv-concat
35source web/table/html.sh # table-sort-{begin,end}
36
37# Also in build/dev-shell.sh
38USER_WEDGE_DIR=~/wedge/oils-for-unix.org
39ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
40
41readonly DEPS_SOURCE_DIR=_build/deps-source
42
43readonly RE2C_VERSION=3.0
44readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
45
46readonly CMARK_VERSION=0.29.0
47readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
48
49readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
50
51readonly PY2_VERSION=2.7.18
52readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
53
54readonly PY3_VERSION=3.10.4
55readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
56
57readonly BASH_VER=4.4 # don't clobber BASH_VERSION
58readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
59
60readonly DASH_VERSION=0.5.10.2
61readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
62
63readonly ZSH_VERSION=5.1.1
64readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
65
66readonly MKSH_VERSION=R52c
67readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
68
69readonly BUSYBOX_VERSION='1.35.0'
70readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
71
72readonly YASH_VERSION=2.49
73readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
74
75readonly MYPY_GIT_URL=https://github.com/python/mypy
76readonly MYPY_VERSION=0.780
77
78readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
79
80# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
81# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
82readonly PYFLAKES_VERSION=2.4.0
83#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
84# 2023-07: Mirrored to avoid network problem on broome during release
85readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
86
87readonly BLOATY_VERSION=1.1
88readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
89
90readonly UFTRACE_VERSION=0.13
91readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
92
93log() {
94 echo "$0: $@" >& 2
95}
96
97die() {
98 log "$@"
99 exit 1
100}
101
102rm-oils-crap() {
103 ### When you want to start over
104
105 rm -r -f -v ~/wedge
106 sudo rm -r -f -v /wedge
107}
108
109# Note: git is an implicit dependency -- that's how we got the repo in the
110# first place!
111
112# python2-dev is no longer available on Debian 12
113# python-dev also seems gone
114#
115# wget: for fetching wedges (not on Debian by default!)
116# tree: tiny package that's useful for showing what we installed
117# g++: essential
118# libreadline-dev: needed for the build/prepare.sh Python build.
119# gawk: used by spec-runner.sh for the special match() function.
120# cmake: for cmark
121# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
122# as well
123readonly -a WEDGE_DEPS_DEBIAN=(
124 bzip2
125 wget
126 tree
127 gawk
128 g++
129 ninja-build
130 cmake
131 libreadline-dev
132 "${PY3_BUILD_DEPS[@]}"
133)
134
135readonly -a WEDGE_DEPS_ALPINE=(
136 bzip2
137 xz
138
139 wget tree gawk
140
141 gcc g++
142 ninja-build
143 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
144 ninja-is-really-ninja
145 cmake
146
147 readline-dev
148 zlib-dev
149 libffi-dev
150 openssl-dev
151
152 ncurses-dev
153)
154
155readonly -a WEDGE_DEPS_FEDORA=(
156
157 # Weird, Fedora doesn't have these by default!
158 hostname
159 tar
160 bzip2
161
162 # https://packages.fedoraproject.org/pkgs/wget/wget/
163 wget
164 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
165 tree
166 gawk
167
168 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
169 gcc gcc-c++
170
171 ninja-build
172 cmake
173
174 readline-devel
175
176 # Like PY3_BUILD_DEPS
177 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
178 zlib-devel
179 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
180 libffi-devel
181 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
182 openssl-devel
183
184 # For building zsh from source?
185 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
186 ncurses-devel
187 #libcap-devel
188
189 # still have a job control error compiling bash
190 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
191 # glibc-devel
192)
193
194install-debian-packages() {
195 ### Packages for build/py.sh all, building wedges, etc.
196
197 set -x # show what needs sudo
198
199 # pass -y for say gitpod
200 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
201 set +x
202
203 # maybe pass -y through
204 test/spec-bin.sh install-shells-with-apt "$@"
205}
206
207install-ubuntu-packages() {
208 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
209 install-debian-packages "$@"
210}
211
212wedge-deps-debian() {
213 # Install packages without prompt
214
215 # 2024-02 - there was an Ubuntu update, and we started needing this
216 sudo apt-get -y update
217
218 install-debian-packages -y
219}
220
221wedge-deps-fedora() {
222 # https://linuxconfig.org/install-development-tools-on-redhat-8
223 # Trying to get past compile errors
224 # sudo dnf group install --assumeyes 'Development Tools'
225
226 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
227}
228
229wedge-deps-alpine() {
230 # https://linuxconfig.org/install-development-tools-on-redhat-8
231 # Trying to get past compile errors
232 # sudo dnf group install --assumeyes 'Development Tools'
233
234 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
235}
236
237#
238# Unused patch, was experiment for Fedora
239#
240
241get-typed-ast-patch() {
242 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
243}
244
245# Work around typed_ast bug:
246# https://github.com/python/typed_ast/issues/169
247#
248# Apply this patch
249# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
250#
251# typed_ast is tarred up though
252patch-typed-ast() {
253 local package_dir=_cache/py3-libs
254 local patch=$PWD/deps/typed_ast.patch
255
256 pushd $package_dir
257 cat $patch
258 echo
259
260 local dir=typed_ast-1.4.3
261 local tar=typed_ast-1.4.3.tar.gz
262
263 echo OLD
264 ls -l $tar
265 echo
266
267 rm -r -f -v $dir
268 tar -x -z < $tar
269
270 pushd $dir
271 patch -p1 < $patch
272 popd
273 #find $dir
274
275 # Create a new one
276 tar --create --gzip --file $tar typed_ast-1.4.3
277
278 echo NEW
279 ls -l $tar
280 echo
281
282 popd
283}
284
285#
286# Fetch
287#
288
289download-to() {
290 local dir=$1
291 local url=$2
292 wget --no-clobber --directory-prefix "$dir" "$url"
293}
294
295maybe-extract() {
296 local wedge_dir=$1
297 local tar_name=$2
298 local out_dir=$3
299
300 if test -d "$wedge_dir/$out_dir"; then
301 log "Not extracting because $wedge_dir/$out_dir exists"
302 return
303 fi
304
305 local tar=$wedge_dir/$tar_name
306 case $tar_name in
307 *.gz|*.tgz) # mksh ends with .tgz
308 flag='--gzip'
309 ;;
310 *.bz2)
311 flag='--bzip2'
312 ;;
313 *.xz)
314 flag='--xz'
315 ;;
316 *)
317 die "tar with unknown extension: $tar_name"
318 ;;
319 esac
320
321 tar --extract $flag --file $tar --directory $wedge_dir
322}
323
324clone-mypy() {
325 ### replaces deps/from-git
326 local dest_dir=$1
327 local version=${2:-$MYPY_VERSION}
328
329 local dest=$dest_dir/mypy-$version
330 if test -d $dest; then
331 log "Not cloning because $dest exists"
332 return
333 fi
334
335 # v$VERSION is a tag, not a branch
336
337 # size optimization: --depth=1 --shallow-submodules
338 # https://git-scm.com/docs/git-clone
339
340 git clone --recursive --branch v$version \
341 --depth=1 --shallow-submodules \
342 $MYPY_GIT_URL $dest
343
344 # TODO: verify commit checksum
345}
346
347copy-source-medo() {
348 mkdir -p $DEPS_SOURCE_DIR
349
350 # Copy the whole tree, including the .treeptr files
351 cp --verbose --recursive --no-target-directory \
352 deps/source.medo/ $DEPS_SOURCE_DIR/
353}
354
355fetch() {
356 local py_only=${1:-}
357
358 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
359 # would do: fetch compressed tarballs designated by .treeptr files, and
360 # expand them.
361
362 # _build/deps-source/
363 # re2c/
364 # WEDGE
365 # re2c-3.0/ # expanded .tar.xz file
366
367 copy-source-medo
368
369 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
370 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
371 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
372 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
373
374 if test -n "$py_only"; then
375 log "Fetched dependencies for 'build/py.sh'"
376 return
377 fi
378
379 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
380 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
381 pyflakes-$PYFLAKES_VERSION
382
383 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
384 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
385 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
386 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
387
388 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
389 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" dash-$BASH_VER
390
391 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
392 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
393
394 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
395 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
396
397 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
398 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
399
400 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
401 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
402
403 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
404 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$DASH_VERSION
405
406 # Patch: this tarball doesn't follow the convention $name-$version
407 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
408 pushd $DEPS_SOURCE_DIR/mksh
409 mv -v mksh mksh-$MKSH_VERSION
410 popd
411 fi
412
413 # bloaty and uftrace are for benchmarks, in containers
414 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
415 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
416 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
417 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
418
419 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
420
421 # It's also copied into a wedge in install-wedges.
422 clone-mypy $DEPS_SOURCE_DIR/mypy
423
424 if command -v tree > /dev/null; then
425 tree -L 2 $DEPS_SOURCE_DIR
426 fi
427}
428
429fetch-py() {
430 fetch py_only
431}
432
433mirror-pyflakes() {
434 ### Workaround for network error during release
435 scp \
436 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
437 oilshell.org:oilshell.org/blob/
438}
439
440wedge-exists() {
441 ### Does an installed wedge already exist?
442
443 local name=$1
444 local version=$2
445 local wedge_dir=${3:-/wedge/oils-for-unix.org}
446
447 local installed=$wedge_dir/pkg/$name/$version
448
449 if test -d $installed; then
450 log "$installed already exists"
451 return 0
452 else
453 return 1
454 fi
455}
456
457#
458# Install
459#
460
461# TODO: py3-libs needs to be a WEDGE, so that that you can run
462# 'wedge build deps/source.medo/py3-libs/' and then get it in
463#
464# _build/wedge/{absolute,relative} # which one?
465#
466# It needs a BUILD DEPENDENCY on:
467# - the python3 wedge, so you can do python3 -m pip install.
468# - the mypy repo, which has test-requirements.txt
469
470download-py3-libs() {
471 ### Download source/binary packages, AFTER python3 is installed
472
473 # Note that this is NOT source code; there is binary code, e.g. in
474 # lxml-*.whl
475
476 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
477 local py_package_dir=_cache/py3-libs
478 mkdir -p $py_package_dir
479
480 # Avoids a warning, but doesn't fix typed_ast
481 #python3 -m pip download -d $py_package_dir wheel
482
483 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
484 python3 -m pip download -d $py_package_dir pexpect
485}
486
487install-py3-libs-in-venv() {
488 local venv_dir=$1
489 local mypy_dir=$2 # This is a param for host build vs. container build
490 local package_dir=_cache/py3-libs
491
492 source $venv_dir/bin/activate # enter virtualenv
493
494 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
495 # conflicts with MyPy deps!
496 # "ERROR: pip's dependency resolver does not currently take into account all
497 # the packages that are installed."
498
499 # --find-links uses a "cache dir" for packages (weird flag name!)
500
501 # Avoids a warning, but doesn't fix typed_ast
502 #time python3 -m pip install --find-links $package_dir wheel
503
504 # for mycpp/
505 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
506
507 # pexpect: for spec/stateful/*.py
508 time python3 -m pip install --find-links $package_dir pexpect
509}
510
511install-py3-libs-from-cache() {
512
513 # As well as end users
514
515 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
516
517 local py3
518 py3=$(command -v python3)
519 case $py3 in
520 *wedge/oils-for-unix.org/*)
521 ;;
522 *)
523 die "python3 is '$py3', but expected it to be in a wedge"
524 ;;
525 esac
526
527 log "Ensuring pip is installed (interpreter $(command -v python3)"
528 python3 -m ensurepip
529
530 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
531 log "Creating venv in $venv_dir"
532
533 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
534 # the /wedge we just built
535 python3 -m venv $venv_dir
536
537 log "Installing MyPy deps in venv"
538
539 # Run in a subshell because it mutates shell state
540 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
541}
542
543install-py3-libs() {
544 ### Invoked by Dockerfile.cpp-small, etc.
545
546 download-py3-libs
547 install-py3-libs-from-cache
548}
549
550# OBSOLETE in favor of install-spec-bin-fast
551install-spec-bin() {
552 if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
553 deps/wedge.sh unboxed-build _build/deps-source/dash
554 fi
555
556 if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
557 deps/wedge.sh unboxed-build _build/deps-source/mksh
558 fi
559
560 if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
561 deps/wedge.sh unboxed-build _build/deps-source/busybox
562 fi
563
564 # Fedora compile error - count_all_jobs
565 if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
566 deps/wedge.sh unboxed-build _build/deps-source/bash
567 fi
568
569 # Fedora compiler error
570 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
571 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
572 # like ncurses-devel should fix this, but it doesn't
573 #
574 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
575 #
576 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
577 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
578 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
579 #
580 # I think the ./configure is out of sync with the actual build?
581
582 if ! wedge-exists zsh $ZSH_VERSION ''; then
583 deps/wedge.sh unboxed-build _build/deps-source/zsh
584 fi
585
586 return
587
588 # Hm this has problem with out-of-tree build? I think Oils does too actually
589 if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
590 deps/wedge.sh unboxed-build _build/deps-source/yash
591 fi
592}
593
594# TODO:
595# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
596# already in each WEDGE file
597
598py-wedges() {
599 ### for build/py.sh all
600
601 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
602 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
603 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
604 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
605}
606
607cpp-wedges() {
608 ### for ninja / mycpp translation
609
610 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
611 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
612
613 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
614 # separately for now
615 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
616}
617
618spec-bin-wedges() {
619 ### for test/spec-py.sh osh-all
620
621 echo dash $DASH_VERSION $USER_WEDGE_DIR
622 echo bash $BASH_VER $USER_WEDGE_DIR
623 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
624 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
625 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
626}
627
628timestamp() {
629 date '+%H:%M:%S'
630}
631
632my-time-tsv() {
633 python3 benchmarks/time_.py \
634 --tsv \
635 --time-span --rusage \
636 "$@"
637}
638
639maybe-install-wedge() {
640 local name=$1
641 local version=$2
642 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
643
644 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
645 local log_file=$WEDGE_LOG_DIR/$name.log.txt
646
647 echo " TASK $(timestamp) $name $version > $log_file"
648
649 # python3 because it's OUTSIDE the container
650 # Separate columns that could be joined: number of files, total size
651 my-time-tsv --print-header \
652 --field xargs_slot \
653 --field wedge \
654 --field wedge_HREF \
655 --field version \
656 --output $task_file
657
658 if wedge-exists "$name" "$version" "$wedge_dir"; then
659 echo "CACHED $(timestamp) $name $version"
660 return
661 fi
662
663 local -a cmd=( deps/wedge.sh unboxed-build _build/deps-source/$name/ )
664
665 set +o errexit
666 my-time-tsv \
667 --field "$XARGS_SLOT" \
668 --field "$name" \
669 --field "$name.log.txt" \
670 --field "$version" \
671 --append \
672 --output $task_file \
673 "${cmd[@]}" "$@" >$log_file 2>&1
674 local status=$?
675 set -o errexit
676
677 if test "$status" -eq 0; then
678 echo " OK $(timestamp) $name $version"
679 else
680 echo " FAIL $(timestamp) $name $version"
681 fi
682}
683
684dummy-task() {
685 ### For testing log capture
686 local name=$1
687 local version=$2
688
689 echo "Building $name $version"
690
691 # random float between 0 and 3
692 # weirdly we need a seed from bash
693 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
694 local secs
695 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
696
697 echo "sleep $secs"
698 sleep $secs
699
700 echo 'stdout'
701 log 'stderr'
702
703 if test $name = 'mksh'; then
704 echo "simulate failure for $name"
705 exit 2
706 fi
707}
708
709readonly WEDGE_LOG_DIR=_build/wedge/logs
710
711dummy-task-wrapper() {
712 # Similar to test/common.sh run-task-with-status, used by
713 # test/{spec,wild}-runner.sh
714 local name=$1
715 local version=$2
716
717 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
718 local log_file=$WEDGE_LOG_DIR/$name.log.txt
719
720 echo " TASK $(timestamp) $name $version > $log_file"
721
722 # python3 because it's OUTSIDE the container
723 # Separate columns that could be joined: number of files, total size
724 my-time-tsv --print-header \
725 --field xargs_slot \
726 --field wedge \
727 --field wedge_HREF \
728 --field version \
729 --output $task_file
730
731 my-time-tsv \
732 --field "$XARGS_SLOT" \
733 --field "$name" \
734 --field "$name.log.txt" \
735 --field "$version" \
736 --append \
737 --output $task_file \
738 $0 dummy-task "$@" >$log_file 2>&1 || true
739
740 echo " DONE $(timestamp) $name $version"
741}
742
743html-head() {
744 # python3 because we're outside containers
745 PYTHONPATH=. python3 doctools/html_head.py "$@"
746}
747
748index-html() {
749 local tasks_tsv=$1
750
751 local base_url='../../../web'
752 html-head --title 'Wedge Builds' \
753 "$base_url/ajax.js" \
754 "$base_url/table/table-sort.js" \
755 "$base_url/table/table-sort.css" \
756 "$base_url/base.css"\
757
758 table-sort-begin 'width60'
759
760 cat <<EOF
761 <p id="home-link">
762 <a href="/">oilshell.org</a>
763 </p>
764
765 <h1>Wedge Builds</h1>
766EOF
767
768 tsv2html3 $tasks_tsv
769
770 cat <<EOF
771 <p>
772 <a href="tasks.tsv">tasks.tsv</a>
773 </p>
774EOF
775
776 table-sort-end 'tasks' # ID for sorting
777}
778
779NPROC=$(nproc)
780#NPROC=1
781
782install-wedge-list() {
783 ### Reads task rows from stdin
784 local parallel=${1:-}
785
786
787 mkdir -p _build/wedge/logs
788
789 local -a flags
790 if test -n "$parallel"; then
791 log ""
792 log "=== Installing wedges with $NPROC jobs in parallel"
793 log ""
794 flags=( -P $NPROC )
795 else
796 log ""
797 log "=== Installing wedges serially"
798 log ""
799 fi
800
801 # Reads from stdin
802 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
803 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
804
805 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
806}
807
808write-task-report() {
809 local tasks_tsv=_build/wedge/logs/tasks.tsv
810
811 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
812 log "Wrote $tasks_tsv"
813
814 # TODO: version can be right-justified?
815 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
816column_name type precision strftime
817status integer 0 -
818elapsed_secs float 1 -
819user_secs float 1 -
820start_time float 1 %H:%M:%S
821end_time float 1 %H:%M:%S
822sys_secs float 1 -
823max_rss_KiB integer 0 -
824xargs_slot integer 0 -
825wedge string 0 -
826wedge_HREF string 0 -
827version string 0 -
828EOF
829
830 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
831 log "Wrote $WEDGE_LOG_DIR/index.html"
832}
833
834install-spec-bin-fast() {
835 spec-bin-wedges | install-wedge-list T
836 write-task-report
837}
838
839fake-py3-libs-wedge() {
840 local name=py3-libs
841 local version=$PY3_LIBS_VERSION
842
843 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
844 local log_file=$WEDGE_LOG_DIR/$name.log.txt
845
846 my-time-tsv --print-header \
847 --field xargs_slot \
848 --field wedge \
849 --field wedge_HREF \
850 --field version \
851 --output $task_file
852
853 # There is no xargs slot!
854 my-time-tsv \
855 --field "-1" \
856 --field "$name" \
857 --field "$name.log.txt" \
858 --field "$version" \
859 --append \
860 --output $task_file \
861 $0 install-py3-libs >$log_file 2>&1 || true
862
863 echo " FAKE $(timestamp) $name $version"
864}
865
866install-wedges-fast() {
867 echo " START $(timestamp)"
868
869 # Do all of them in parallel
870 { py-wedges; cpp-wedges; spec-bin-wedges; } | install-wedge-list T
871
872 fake-py3-libs-wedge
873 echo " END $(timestamp)"
874
875 write-task-report
876}
877
878# OBSOLETE in favor of install-wedges-fast
879install-wedges() {
880 local py_only=${1:-}
881
882 # TODO:
883 # - Make all of these RELATIVE wedges
884 # - Add
885 # - unboxed-rel-smoke-test -- move it inside container
886 # - rel-smoke-test -- mount it in a different location
887
888 if ! wedge-exists cmark $CMARK_VERSION; then
889 deps/wedge.sh unboxed-build _build/deps-source/cmark/
890 fi
891
892 if ! wedge-exists re2c $RE2C_VERSION; then
893 deps/wedge.sh unboxed-build _build/deps-source/re2c/
894 fi
895
896 if ! wedge-exists python2 $PY2_VERSION; then
897 deps/wedge.sh unboxed-build _build/deps-source/python2/
898 fi
899
900 if test -n "$py_only"; then
901 log "Installed dependencies for 'build/py.sh'"
902 return
903 fi
904
905 # Just copy this source tarball
906 if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
907 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
908 mkdir -p $dest_dir
909
910 cp --verbose --recursive --no-target-directory \
911 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
912 fi
913
914 if ! wedge-exists python3 $PY3_VERSION; then
915 deps/wedge.sh unboxed-build _build/deps-source/python3/
916 fi
917
918 # Copy all the contents, except for .git folder.
919 if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
920
921 # NOTE: We have to also copy the .git dir, because it has
922 # .git/modules/typeshed
923 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
924 mkdir -p $dest_dir
925
926 # Note: pack files in .git/modules/typeshed/objects/pack are read-only
927 # this can fail
928 cp --verbose --recursive --no-target-directory \
929 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
930 fi
931
932 if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
933 download-py3-libs
934 # This patch doesn't work?
935 # patch-typed-ast
936 install-py3-libs
937 fi
938
939 if command -v tree > /dev/null; then
940 tree -L 3 $USER_WEDGE_DIR
941 echo
942 tree -L 3 /wedge/oils-for-unix.org
943 fi
944}
945
946install-wedges-py() {
947 install-wedges py_only
948}
949
950#
951# Unboxed wedge builds
952#
953
954uftrace-host() {
955 ### built on demand; run $0 first
956
957 # BUG: doesn't detect python3
958 # WEDGE tells me that it depends on pkg-config
959 # 'apt-get install pkgconf' gets it
960 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
961
962 deps/wedge.sh unboxed-build _build/deps-source/uftrace
963}
964
965bloaty-host() {
966 deps/wedge.sh unboxed-build _build/deps-source/bloaty
967}
968
969R-libs-host() {
970 deps/wedge.sh unboxed-build _build/deps-source/R-libs
971}
972
973#
974# Wedges built inside a container, for copying into a container
975#
976
977container-wedges() {
978 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
979
980 #export-podman
981
982 if true; then
983 deps/wedge.sh build deps/source.medo/time-helper
984 deps/wedge.sh build deps/source.medo/cmark/
985 deps/wedge.sh build deps/source.medo/re2c/
986 deps/wedge.sh build deps/source.medo/python3/
987 fi
988
989 if false; then
990 deps/wedge.sh build deps/source.medo/bloaty/
991 deps/wedge.sh build deps/source.medo/uftrace/
992 fi
993
994 if false; then
995 # For soil-benchmarks/ images
996 deps/wedge.sh build deps/source.medo/R-libs/
997 fi
998}
999
1000#
1001# Report
1002#
1003
1004commas() {
1005 # Wow I didn't know this :a trick
1006 #
1007 # OK this is a label and a loop, which makes sense. You can't do it with
1008 # pure regex.
1009 #
1010 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1011 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1012 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1013}
1014
1015wedge-sizes() {
1016 local tmp=_tmp/wedge-sizes.txt
1017
1018 # -b is --bytes, but use short flag for busybox compat
1019 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1020 { print $0 # print the line
1021 total_bytes += $1 # accumulate
1022 }
1023END { print total_bytes " TOTAL" }
1024' > $tmp
1025
1026 # printf justifies du output
1027 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1028 echo
1029
1030 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1031 #echo
1032}
1033
1034wedge-report() {
1035 # 4 levels deep shows the package
1036 if command -v tree > /dev/null; then
1037 tree -L 4 /wedge ~/wedge
1038 echo
1039 fi
1040
1041 wedge-sizes
1042
1043 local tmp=_tmp/wedge-manifest.txt
1044
1045 echo 'Biggest files'
1046 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1047 # busybox find doesn't have -printf
1048 echo 'find -printf failed'
1049 return
1050 fi
1051
1052 set +o errexit # ignore SIGPIPE
1053 sort -n --reverse $tmp | head -n 20 | commas
1054 set -o errexit
1055
1056 echo
1057
1058 # Show the most common file extensions
1059 #
1060 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1061 # most common
1062 #
1063 # There are also duplicate .a files for Python -- should look at how distros
1064 # get rid of those
1065
1066 cat $tmp | python3 -c '
1067import os, sys, collections
1068
1069bytes = collections.Counter()
1070files = collections.Counter()
1071
1072for line in sys.stdin:
1073 size, path = line.split(None, 1)
1074 path = path.strip() # remove newline
1075 _, ext = os.path.splitext(path)
1076 size = int(size)
1077
1078 bytes[ext] += size
1079 files[ext] += 1
1080
1081#print(bytes)
1082#print(files)
1083
1084n = 20
1085
1086print("Most common file types")
1087for ext, count in files.most_common()[:n]:
1088 print("%10d %s" % (count, ext))
1089
1090print()
1091
1092print("Total bytes by file type")
1093for ext, total_bytes in bytes.most_common()[:n]:
1094 print("%10d %s" % (total_bytes, ext))
1095' | commas
1096}
1097
1098run-task "$@"