OILS / build / deps.sh View on Github | oilshell.org

1099 lines, 578 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
13#
14# TODO: Do we need something faster, just python2, re2c, and cmark?
15#
16# - build/deps.sh fetch-py
17# - build/deps.sh install-wedges-py
18#
19# TODO: Can we make most of them non-root deps? This requires rebuilding
20# containers, which requires podman.
21#
22# rm -r -f ~/wedge # would be better
23
24set -o nounset
25set -o pipefail
26set -o errexit
27
28REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
29
30source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
31source deps/from-apt.sh # PY3_BUILD_DEPS
32#source deps/podman.sh
33source devtools/run-task.sh # run-task
34source test/tsv-lib.sh # tsv-concat
35source web/table/html.sh # table-sort-{begin,end}
36
37# Also in build/dev-shell.sh
38USER_WEDGE_DIR=~/wedge/oils-for-unix.org
39ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
40
41readonly DEPS_SOURCE_DIR=_build/deps-source
42
43readonly RE2C_VERSION=3.0
44readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
45
46readonly CMARK_VERSION=0.29.0
47readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
48
49readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
50
51readonly PY2_VERSION=2.7.18
52readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
53
54readonly PY3_VERSION=3.10.4
55readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
56
57readonly BASH_VER=4.4 # don't clobber BASH_VERSION
58readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
59
60readonly DASH_VERSION=0.5.10.2
61readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
62
63readonly ZSH_VERSION=5.1.1
64readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
65
66readonly MKSH_VERSION=R52c
67readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
68
69readonly BUSYBOX_VERSION='1.35.0'
70readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
71
72readonly YASH_VERSION=2.49
73readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
74
75readonly MYPY_GIT_URL=https://github.com/python/mypy
76readonly MYPY_VERSION=0.780
77
78readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
79
80# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
81# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
82readonly PYFLAKES_VERSION=2.4.0
83#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
84# 2023-07: Mirrored to avoid network problem on broome during release
85readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
86
87readonly BLOATY_VERSION=1.1
88readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
89
90readonly UFTRACE_VERSION=0.13
91readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
92
93log() {
94 echo "$0: $@" >& 2
95}
96
97die() {
98 log "$@"
99 exit 1
100}
101
102rm-oils-crap() {
103 ### When you want to start over
104
105 rm -r -f -v ~/wedge
106 sudo rm -r -f -v /wedge
107}
108
109# Note: git is an implicit dependency -- that's how we got the repo in the
110# first place!
111
112# python2-dev is no longer available on Debian 12
113# python-dev also seems gone
114#
115# wget: for fetching wedges (not on Debian by default!)
116# tree: tiny package that's useful for showing what we installed
117# g++: essential
118# libreadline-dev: needed for the build/prepare.sh Python build.
119# gawk: used by spec-runner.sh for the special match() function.
120# cmake: for cmark
121# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
122# as well
123readonly -a WEDGE_DEPS_DEBIAN=(
124 bzip2
125 wget
126 tree
127 gawk
128 g++
129 ninja-build
130 cmake
131 libreadline-dev
132 systemtap-sdt-dev
133 "${PY3_BUILD_DEPS[@]}"
134)
135
136readonly -a WEDGE_DEPS_ALPINE=(
137 bzip2
138 xz
139
140 wget tree gawk
141
142 gcc g++
143 ninja-build
144 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
145 ninja-is-really-ninja
146 cmake
147
148 readline-dev
149 zlib-dev
150 libffi-dev
151 openssl-dev
152
153 ncurses-dev
154)
155
156readonly -a WEDGE_DEPS_FEDORA=(
157
158 # Weird, Fedora doesn't have these by default!
159 hostname
160 tar
161 bzip2
162
163 # https://packages.fedoraproject.org/pkgs/wget/wget/
164 wget
165 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
166 tree
167 gawk
168
169 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
170 gcc gcc-c++
171
172 ninja-build
173 cmake
174
175 readline-devel
176
177 # Like PY3_BUILD_DEPS
178 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
179 zlib-devel
180 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
181 libffi-devel
182 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
183 openssl-devel
184
185 # For building zsh from source?
186 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
187 ncurses-devel
188 #libcap-devel
189
190 # still have a job control error compiling bash
191 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
192 # glibc-devel
193)
194
195install-debian-packages() {
196 ### Packages for build/py.sh all, building wedges, etc.
197
198 set -x # show what needs sudo
199
200 # pass -y for say gitpod
201 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
202 set +x
203
204 # maybe pass -y through
205 test/spec-bin.sh install-shells-with-apt "$@"
206}
207
208install-ubuntu-packages() {
209 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
210 install-debian-packages "$@"
211}
212
213wedge-deps-debian() {
214 # Install packages without prompt
215
216 # 2024-02 - there was an Ubuntu update, and we started needing this
217 sudo apt-get -y update
218
219 install-debian-packages -y
220}
221
222wedge-deps-fedora() {
223 # https://linuxconfig.org/install-development-tools-on-redhat-8
224 # Trying to get past compile errors
225 # sudo dnf group install --assumeyes 'Development Tools'
226
227 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
228}
229
230wedge-deps-alpine() {
231 # https://linuxconfig.org/install-development-tools-on-redhat-8
232 # Trying to get past compile errors
233 # sudo dnf group install --assumeyes 'Development Tools'
234
235 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
236}
237
238#
239# Unused patch, was experiment for Fedora
240#
241
242get-typed-ast-patch() {
243 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
244}
245
246# Work around typed_ast bug:
247# https://github.com/python/typed_ast/issues/169
248#
249# Apply this patch
250# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
251#
252# typed_ast is tarred up though
253patch-typed-ast() {
254 local package_dir=_cache/py3-libs
255 local patch=$PWD/deps/typed_ast.patch
256
257 pushd $package_dir
258 cat $patch
259 echo
260
261 local dir=typed_ast-1.4.3
262 local tar=typed_ast-1.4.3.tar.gz
263
264 echo OLD
265 ls -l $tar
266 echo
267
268 rm -r -f -v $dir
269 tar -x -z < $tar
270
271 pushd $dir
272 patch -p1 < $patch
273 popd
274 #find $dir
275
276 # Create a new one
277 tar --create --gzip --file $tar typed_ast-1.4.3
278
279 echo NEW
280 ls -l $tar
281 echo
282
283 popd
284}
285
286#
287# Fetch
288#
289
290download-to() {
291 local dir=$1
292 local url=$2
293 wget --no-clobber --directory-prefix "$dir" "$url"
294}
295
296maybe-extract() {
297 local wedge_dir=$1
298 local tar_name=$2
299 local out_dir=$3
300
301 if test -d "$wedge_dir/$out_dir"; then
302 log "Not extracting because $wedge_dir/$out_dir exists"
303 return
304 fi
305
306 local tar=$wedge_dir/$tar_name
307 case $tar_name in
308 *.gz|*.tgz) # mksh ends with .tgz
309 flag='--gzip'
310 ;;
311 *.bz2)
312 flag='--bzip2'
313 ;;
314 *.xz)
315 flag='--xz'
316 ;;
317 *)
318 die "tar with unknown extension: $tar_name"
319 ;;
320 esac
321
322 tar --extract $flag --file $tar --directory $wedge_dir
323}
324
325clone-mypy() {
326 ### replaces deps/from-git
327 local dest_dir=$1
328 local version=${2:-$MYPY_VERSION}
329
330 local dest=$dest_dir/mypy-$version
331 if test -d $dest; then
332 log "Not cloning because $dest exists"
333 return
334 fi
335
336 # v$VERSION is a tag, not a branch
337
338 # size optimization: --depth=1 --shallow-submodules
339 # https://git-scm.com/docs/git-clone
340
341 git clone --recursive --branch v$version \
342 --depth=1 --shallow-submodules \
343 $MYPY_GIT_URL $dest
344
345 # TODO: verify commit checksum
346}
347
348copy-source-medo() {
349 mkdir -p $DEPS_SOURCE_DIR
350
351 # Copy the whole tree, including the .treeptr files
352 cp --verbose --recursive --no-target-directory \
353 deps/source.medo/ $DEPS_SOURCE_DIR/
354}
355
356fetch() {
357 local py_only=${1:-}
358
359 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
360 # would do: fetch compressed tarballs designated by .treeptr files, and
361 # expand them.
362
363 # _build/deps-source/
364 # re2c/
365 # WEDGE
366 # re2c-3.0/ # expanded .tar.xz file
367
368 copy-source-medo
369
370 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
371 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
372 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
373 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
374
375 if test -n "$py_only"; then
376 log "Fetched dependencies for 'build/py.sh'"
377 return
378 fi
379
380 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
381 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
382 pyflakes-$PYFLAKES_VERSION
383
384 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
385 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
386 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
387 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
388
389 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
390 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" dash-$BASH_VER
391
392 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
393 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
394
395 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
396 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
397
398 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
399 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
400
401 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
402 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
403
404 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
405 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$DASH_VERSION
406
407 # Patch: this tarball doesn't follow the convention $name-$version
408 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
409 pushd $DEPS_SOURCE_DIR/mksh
410 mv -v mksh mksh-$MKSH_VERSION
411 popd
412 fi
413
414 # bloaty and uftrace are for benchmarks, in containers
415 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
416 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
417 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
418 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
419
420 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
421
422 # It's also copied into a wedge in install-wedges.
423 clone-mypy $DEPS_SOURCE_DIR/mypy
424
425 if command -v tree > /dev/null; then
426 tree -L 2 $DEPS_SOURCE_DIR
427 fi
428}
429
430fetch-py() {
431 fetch py_only
432}
433
434mirror-pyflakes() {
435 ### Workaround for network error during release
436 scp \
437 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
438 oilshell.org:oilshell.org/blob/
439}
440
441wedge-exists() {
442 ### Does an installed wedge already exist?
443
444 local name=$1
445 local version=$2
446 local wedge_dir=${3:-/wedge/oils-for-unix.org}
447
448 local installed=$wedge_dir/pkg/$name/$version
449
450 if test -d $installed; then
451 log "$installed already exists"
452 return 0
453 else
454 return 1
455 fi
456}
457
458#
459# Install
460#
461
462# TODO: py3-libs needs to be a WEDGE, so that that you can run
463# 'wedge build deps/source.medo/py3-libs/' and then get it in
464#
465# _build/wedge/{absolute,relative} # which one?
466#
467# It needs a BUILD DEPENDENCY on:
468# - the python3 wedge, so you can do python3 -m pip install.
469# - the mypy repo, which has test-requirements.txt
470
471download-py3-libs() {
472 ### Download source/binary packages, AFTER python3 is installed
473
474 # Note that this is NOT source code; there is binary code, e.g. in
475 # lxml-*.whl
476
477 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
478 local py_package_dir=_cache/py3-libs
479 mkdir -p $py_package_dir
480
481 # Avoids a warning, but doesn't fix typed_ast
482 #python3 -m pip download -d $py_package_dir wheel
483
484 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
485 python3 -m pip download -d $py_package_dir pexpect
486}
487
488install-py3-libs-in-venv() {
489 local venv_dir=$1
490 local mypy_dir=$2 # This is a param for host build vs. container build
491 local package_dir=_cache/py3-libs
492
493 source $venv_dir/bin/activate # enter virtualenv
494
495 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
496 # conflicts with MyPy deps!
497 # "ERROR: pip's dependency resolver does not currently take into account all
498 # the packages that are installed."
499
500 # --find-links uses a "cache dir" for packages (weird flag name!)
501
502 # Avoids a warning, but doesn't fix typed_ast
503 #time python3 -m pip install --find-links $package_dir wheel
504
505 # for mycpp/
506 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
507
508 # pexpect: for spec/stateful/*.py
509 time python3 -m pip install --find-links $package_dir pexpect
510}
511
512install-py3-libs-from-cache() {
513
514 # As well as end users
515
516 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
517
518 local py3
519 py3=$(command -v python3)
520 case $py3 in
521 *wedge/oils-for-unix.org/*)
522 ;;
523 *)
524 die "python3 is '$py3', but expected it to be in a wedge"
525 ;;
526 esac
527
528 log "Ensuring pip is installed (interpreter $(command -v python3)"
529 python3 -m ensurepip
530
531 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
532 log "Creating venv in $venv_dir"
533
534 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
535 # the /wedge we just built
536 python3 -m venv $venv_dir
537
538 log "Installing MyPy deps in venv"
539
540 # Run in a subshell because it mutates shell state
541 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
542}
543
544install-py3-libs() {
545 ### Invoked by Dockerfile.cpp-small, etc.
546
547 download-py3-libs
548 install-py3-libs-from-cache
549}
550
551# OBSOLETE in favor of install-spec-bin-fast
552install-spec-bin() {
553 if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
554 deps/wedge.sh unboxed-build _build/deps-source/dash
555 fi
556
557 if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
558 deps/wedge.sh unboxed-build _build/deps-source/mksh
559 fi
560
561 if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
562 deps/wedge.sh unboxed-build _build/deps-source/busybox
563 fi
564
565 # Fedora compile error - count_all_jobs
566 if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
567 deps/wedge.sh unboxed-build _build/deps-source/bash
568 fi
569
570 # Fedora compiler error
571 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
572 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
573 # like ncurses-devel should fix this, but it doesn't
574 #
575 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
576 #
577 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
578 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
579 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
580 #
581 # I think the ./configure is out of sync with the actual build?
582
583 if ! wedge-exists zsh $ZSH_VERSION ''; then
584 deps/wedge.sh unboxed-build _build/deps-source/zsh
585 fi
586
587 return
588
589 # Hm this has problem with out-of-tree build? I think Oils does too actually
590 if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
591 deps/wedge.sh unboxed-build _build/deps-source/yash
592 fi
593}
594
595# TODO:
596# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
597# already in each WEDGE file
598
599py-wedges() {
600 ### for build/py.sh all
601
602 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
603 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
604 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
605 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
606}
607
608cpp-wedges() {
609 ### for ninja / mycpp translation
610
611 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
612 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
613
614 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
615 # separately for now
616 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
617}
618
619spec-bin-wedges() {
620 ### for test/spec-py.sh osh-all
621
622 echo dash $DASH_VERSION $USER_WEDGE_DIR
623 echo bash $BASH_VER $USER_WEDGE_DIR
624 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
625 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
626 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
627}
628
629timestamp() {
630 date '+%H:%M:%S'
631}
632
633my-time-tsv() {
634 python3 benchmarks/time_.py \
635 --tsv \
636 --time-span --rusage \
637 "$@"
638}
639
640maybe-install-wedge() {
641 local name=$1
642 local version=$2
643 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
644
645 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
646 local log_file=$WEDGE_LOG_DIR/$name.log.txt
647
648 echo " TASK $(timestamp) $name $version > $log_file"
649
650 # python3 because it's OUTSIDE the container
651 # Separate columns that could be joined: number of files, total size
652 my-time-tsv --print-header \
653 --field xargs_slot \
654 --field wedge \
655 --field wedge_HREF \
656 --field version \
657 --output $task_file
658
659 if wedge-exists "$name" "$version" "$wedge_dir"; then
660 echo "CACHED $(timestamp) $name $version"
661 return
662 fi
663
664 local -a cmd=( deps/wedge.sh unboxed-build _build/deps-source/$name/ )
665
666 set +o errexit
667 my-time-tsv \
668 --field "$XARGS_SLOT" \
669 --field "$name" \
670 --field "$name.log.txt" \
671 --field "$version" \
672 --append \
673 --output $task_file \
674 "${cmd[@]}" "$@" >$log_file 2>&1
675 local status=$?
676 set -o errexit
677
678 if test "$status" -eq 0; then
679 echo " OK $(timestamp) $name $version"
680 else
681 echo " FAIL $(timestamp) $name $version"
682 fi
683}
684
685dummy-task() {
686 ### For testing log capture
687 local name=$1
688 local version=$2
689
690 echo "Building $name $version"
691
692 # random float between 0 and 3
693 # weirdly we need a seed from bash
694 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
695 local secs
696 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
697
698 echo "sleep $secs"
699 sleep $secs
700
701 echo 'stdout'
702 log 'stderr'
703
704 if test $name = 'mksh'; then
705 echo "simulate failure for $name"
706 exit 2
707 fi
708}
709
710readonly WEDGE_LOG_DIR=_build/wedge/logs
711
712dummy-task-wrapper() {
713 # Similar to test/common.sh run-task-with-status, used by
714 # test/{spec,wild}-runner.sh
715 local name=$1
716 local version=$2
717
718 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
719 local log_file=$WEDGE_LOG_DIR/$name.log.txt
720
721 echo " TASK $(timestamp) $name $version > $log_file"
722
723 # python3 because it's OUTSIDE the container
724 # Separate columns that could be joined: number of files, total size
725 my-time-tsv --print-header \
726 --field xargs_slot \
727 --field wedge \
728 --field wedge_HREF \
729 --field version \
730 --output $task_file
731
732 my-time-tsv \
733 --field "$XARGS_SLOT" \
734 --field "$name" \
735 --field "$name.log.txt" \
736 --field "$version" \
737 --append \
738 --output $task_file \
739 $0 dummy-task "$@" >$log_file 2>&1 || true
740
741 echo " DONE $(timestamp) $name $version"
742}
743
744html-head() {
745 # python3 because we're outside containers
746 PYTHONPATH=. python3 doctools/html_head.py "$@"
747}
748
749index-html() {
750 local tasks_tsv=$1
751
752 local base_url='../../../web'
753 html-head --title 'Wedge Builds' \
754 "$base_url/ajax.js" \
755 "$base_url/table/table-sort.js" \
756 "$base_url/table/table-sort.css" \
757 "$base_url/base.css"\
758
759 table-sort-begin 'width60'
760
761 cat <<EOF
762 <p id="home-link">
763 <a href="/">oilshell.org</a>
764 </p>
765
766 <h1>Wedge Builds</h1>
767EOF
768
769 tsv2html3 $tasks_tsv
770
771 cat <<EOF
772 <p>
773 <a href="tasks.tsv">tasks.tsv</a>
774 </p>
775EOF
776
777 table-sort-end 'tasks' # ID for sorting
778}
779
780NPROC=$(nproc)
781#NPROC=1
782
783install-wedge-list() {
784 ### Reads task rows from stdin
785 local parallel=${1:-}
786
787
788 mkdir -p _build/wedge/logs
789
790 local -a flags
791 if test -n "$parallel"; then
792 log ""
793 log "=== Installing wedges with $NPROC jobs in parallel"
794 log ""
795 flags=( -P $NPROC )
796 else
797 log ""
798 log "=== Installing wedges serially"
799 log ""
800 fi
801
802 # Reads from stdin
803 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
804 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
805
806 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
807}
808
809write-task-report() {
810 local tasks_tsv=_build/wedge/logs/tasks.tsv
811
812 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
813 log "Wrote $tasks_tsv"
814
815 # TODO: version can be right-justified?
816 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
817column_name type precision strftime
818status integer 0 -
819elapsed_secs float 1 -
820user_secs float 1 -
821start_time float 1 %H:%M:%S
822end_time float 1 %H:%M:%S
823sys_secs float 1 -
824max_rss_KiB integer 0 -
825xargs_slot integer 0 -
826wedge string 0 -
827wedge_HREF string 0 -
828version string 0 -
829EOF
830
831 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
832 log "Wrote $WEDGE_LOG_DIR/index.html"
833}
834
835install-spec-bin-fast() {
836 spec-bin-wedges | install-wedge-list T
837 write-task-report
838}
839
840fake-py3-libs-wedge() {
841 local name=py3-libs
842 local version=$PY3_LIBS_VERSION
843
844 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
845 local log_file=$WEDGE_LOG_DIR/$name.log.txt
846
847 my-time-tsv --print-header \
848 --field xargs_slot \
849 --field wedge \
850 --field wedge_HREF \
851 --field version \
852 --output $task_file
853
854 # There is no xargs slot!
855 my-time-tsv \
856 --field "-1" \
857 --field "$name" \
858 --field "$name.log.txt" \
859 --field "$version" \
860 --append \
861 --output $task_file \
862 $0 install-py3-libs >$log_file 2>&1 || true
863
864 echo " FAKE $(timestamp) $name $version"
865}
866
867install-wedges-fast() {
868 echo " START $(timestamp)"
869
870 # Do all of them in parallel
871 { py-wedges; cpp-wedges; spec-bin-wedges; } | install-wedge-list T
872
873 fake-py3-libs-wedge
874 echo " END $(timestamp)"
875
876 write-task-report
877}
878
879# OBSOLETE in favor of install-wedges-fast
880install-wedges() {
881 local py_only=${1:-}
882
883 # TODO:
884 # - Make all of these RELATIVE wedges
885 # - Add
886 # - unboxed-rel-smoke-test -- move it inside container
887 # - rel-smoke-test -- mount it in a different location
888
889 if ! wedge-exists cmark $CMARK_VERSION; then
890 deps/wedge.sh unboxed-build _build/deps-source/cmark/
891 fi
892
893 if ! wedge-exists re2c $RE2C_VERSION; then
894 deps/wedge.sh unboxed-build _build/deps-source/re2c/
895 fi
896
897 if ! wedge-exists python2 $PY2_VERSION; then
898 deps/wedge.sh unboxed-build _build/deps-source/python2/
899 fi
900
901 if test -n "$py_only"; then
902 log "Installed dependencies for 'build/py.sh'"
903 return
904 fi
905
906 # Just copy this source tarball
907 if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
908 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
909 mkdir -p $dest_dir
910
911 cp --verbose --recursive --no-target-directory \
912 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
913 fi
914
915 if ! wedge-exists python3 $PY3_VERSION; then
916 deps/wedge.sh unboxed-build _build/deps-source/python3/
917 fi
918
919 # Copy all the contents, except for .git folder.
920 if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
921
922 # NOTE: We have to also copy the .git dir, because it has
923 # .git/modules/typeshed
924 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
925 mkdir -p $dest_dir
926
927 # Note: pack files in .git/modules/typeshed/objects/pack are read-only
928 # this can fail
929 cp --verbose --recursive --no-target-directory \
930 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
931 fi
932
933 if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
934 download-py3-libs
935 # This patch doesn't work?
936 # patch-typed-ast
937 install-py3-libs
938 fi
939
940 if command -v tree > /dev/null; then
941 tree -L 3 $USER_WEDGE_DIR
942 echo
943 tree -L 3 /wedge/oils-for-unix.org
944 fi
945}
946
947install-wedges-py() {
948 install-wedges py_only
949}
950
951#
952# Unboxed wedge builds
953#
954
955uftrace-host() {
956 ### built on demand; run $0 first
957
958 # BUG: doesn't detect python3
959 # WEDGE tells me that it depends on pkg-config
960 # 'apt-get install pkgconf' gets it
961 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
962
963 deps/wedge.sh unboxed-build _build/deps-source/uftrace
964}
965
966bloaty-host() {
967 deps/wedge.sh unboxed-build _build/deps-source/bloaty
968}
969
970R-libs-host() {
971 deps/wedge.sh unboxed-build _build/deps-source/R-libs
972}
973
974#
975# Wedges built inside a container, for copying into a container
976#
977
978container-wedges() {
979 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
980
981 #export-podman
982
983 if true; then
984 deps/wedge.sh build deps/source.medo/time-helper
985 deps/wedge.sh build deps/source.medo/cmark/
986 deps/wedge.sh build deps/source.medo/re2c/
987 deps/wedge.sh build deps/source.medo/python3/
988 fi
989
990 if false; then
991 deps/wedge.sh build deps/source.medo/bloaty/
992 deps/wedge.sh build deps/source.medo/uftrace/
993 fi
994
995 if false; then
996 # For soil-benchmarks/ images
997 deps/wedge.sh build deps/source.medo/R-libs/
998 fi
999}
1000
1001#
1002# Report
1003#
1004
1005commas() {
1006 # Wow I didn't know this :a trick
1007 #
1008 # OK this is a label and a loop, which makes sense. You can't do it with
1009 # pure regex.
1010 #
1011 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1012 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1013 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1014}
1015
1016wedge-sizes() {
1017 local tmp=_tmp/wedge-sizes.txt
1018
1019 # -b is --bytes, but use short flag for busybox compat
1020 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1021 { print $0 # print the line
1022 total_bytes += $1 # accumulate
1023 }
1024END { print total_bytes " TOTAL" }
1025' > $tmp
1026
1027 # printf justifies du output
1028 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1029 echo
1030
1031 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1032 #echo
1033}
1034
1035wedge-report() {
1036 # 4 levels deep shows the package
1037 if command -v tree > /dev/null; then
1038 tree -L 4 /wedge ~/wedge
1039 echo
1040 fi
1041
1042 wedge-sizes
1043
1044 local tmp=_tmp/wedge-manifest.txt
1045
1046 echo 'Biggest files'
1047 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1048 # busybox find doesn't have -printf
1049 echo 'find -printf failed'
1050 return
1051 fi
1052
1053 set +o errexit # ignore SIGPIPE
1054 sort -n --reverse $tmp | head -n 20 | commas
1055 set -o errexit
1056
1057 echo
1058
1059 # Show the most common file extensions
1060 #
1061 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1062 # most common
1063 #
1064 # There are also duplicate .a files for Python -- should look at how distros
1065 # get rid of those
1066
1067 cat $tmp | python3 -c '
1068import os, sys, collections
1069
1070bytes = collections.Counter()
1071files = collections.Counter()
1072
1073for line in sys.stdin:
1074 size, path = line.split(None, 1)
1075 path = path.strip() # remove newline
1076 _, ext = os.path.splitext(path)
1077 size = int(size)
1078
1079 bytes[ext] += size
1080 files[ext] += 1
1081
1082#print(bytes)
1083#print(files)
1084
1085n = 20
1086
1087print("Most common file types")
1088for ext, count in files.most_common()[:n]:
1089 print("%10d %s" % (count, ext))
1090
1091print()
1092
1093print("Total bytes by file type")
1094for ext, total_bytes in bytes.most_common()[:n]:
1095 print("%10d %s" % (total_bytes, ext))
1096' | commas
1097}
1098
1099run-task "$@"