OILS / build / deps.sh View on Github | oilshell.org

1093 lines, 572 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
13#
14# TODO: Do we need something faster, just python2, re2c, and cmark?
15#
16# - build/deps.sh fetch-py
17# - build/deps.sh install-wedges-py
18#
19# TODO: Can we make most of them non-root deps? This requires rebuilding
20# containers, which requires podman.
21#
22# rm -r -f ~/wedge # would be better
23
24set -o nounset
25set -o pipefail
26set -o errexit
27
28REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
29
30source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
31source deps/from-apt.sh # PY3_BUILD_DEPS
32#source deps/podman.sh
33source devtools/run-task.sh # run-task
34source test/tsv-lib.sh # tsv-concat
35source web/table/html.sh # table-sort-{begin,end}
36
37# Also in build/dev-shell.sh
38USER_WEDGE_DIR=~/wedge/oils-for-unix.org
39ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
40
41readonly DEPS_SOURCE_DIR=_build/deps-source
42
43readonly RE2C_VERSION=3.0
44readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
45
46readonly CMARK_VERSION=0.29.0
47readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
48
49readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
50
51readonly PY2_VERSION=2.7.18
52readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
53
54readonly PY3_VERSION=3.10.4
55readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
56
57readonly BASH_VER=4.4 # don't clobber BASH_VERSION
58readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
59
60readonly DASH_VERSION=0.5.10.2
61readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
62
63readonly ZSH_VERSION=5.1.1
64readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
65
66readonly MKSH_VERSION=R52c
67readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
68
69readonly BUSYBOX_VERSION='1.35.0'
70readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
71
72readonly YASH_VERSION=2.49
73readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
74
75readonly MYPY_GIT_URL=https://github.com/python/mypy
76readonly MYPY_VERSION=0.780
77
78readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
79
80# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
81# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
82readonly PYFLAKES_VERSION=2.4.0
83#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
84# 2023-07: Mirrored to avoid network problem on broome during release
85readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
86
87readonly BLOATY_VERSION=1.1
88readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
89
90readonly UFTRACE_VERSION=0.13
91readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
92
93log() {
94 echo "$0: $@" >& 2
95}
96
97die() {
98 log "$@"
99 exit 1
100}
101
102rm-oils-crap() {
103 ### When you want to start over
104
105 rm -r -f -v ~/wedge
106 sudo rm -r -f -v /wedge
107}
108
109# Note: git is an implicit dependency -- that's how we got the repo in the
110# first place!
111
112# python2-dev is no longer available on Debian 12
113# python-dev also seems gone
114#
115# wget: for fetching wedges (not on Debian by default!)
116# tree: tiny package that's useful for showing what we installed
117# g++: essential
118# libreadline-dev: needed for the build/prepare.sh Python build.
119# gawk: used by spec-runner.sh for the special match() function.
120# cmake: for cmark
121# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
122# as well
123readonly -a WEDGE_DEPS_DEBIAN=(
124 wget tree gawk
125 g++ ninja-build cmake
126 libreadline-dev
127 "${PY3_BUILD_DEPS[@]}"
128)
129
130readonly -a WEDGE_DEPS_ALPINE=(
131 bzip2
132 xz
133
134 wget tree gawk
135
136 gcc g++
137 ninja-build
138 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
139 ninja-is-really-ninja
140 cmake
141
142 readline-dev
143 zlib-dev
144 libffi-dev
145 openssl-dev
146
147 ncurses-dev
148)
149
150readonly -a WEDGE_DEPS_FEDORA=(
151
152 # Weird, Fedora doesn't have these by default!
153 hostname
154 tar
155 bzip2
156
157 # https://packages.fedoraproject.org/pkgs/wget/wget/
158 wget
159 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
160 tree
161 gawk
162
163 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
164 gcc gcc-c++
165
166 ninja-build
167 cmake
168
169 readline-devel
170
171 # Like PY3_BUILD_DEPS
172 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
173 zlib-devel
174 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
175 libffi-devel
176 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
177 openssl-devel
178
179 # For building zsh from source?
180 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
181 ncurses-devel
182 #libcap-devel
183
184 # still have a job control error compiling bash
185 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
186 # glibc-devel
187)
188
189install-debian-packages() {
190 ### Packages for build/py.sh all, building wedges, etc.
191
192 set -x # show what needs sudo
193
194 # pass -y for say gitpod
195 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
196 set +x
197
198 # maybe pass -y through
199 test/spec-bin.sh install-shells-with-apt "$@"
200}
201
202install-ubuntu-packages() {
203 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
204 install-debian-packages "$@"
205}
206
207wedge-deps-debian() {
208 # Install packages without prompt
209
210 # 2024-02 - there was an Ubuntu update, and we started needing this
211 sudo apt-get -y update
212
213 install-debian-packages -y
214}
215
216wedge-deps-fedora() {
217 # https://linuxconfig.org/install-development-tools-on-redhat-8
218 # Trying to get past compile errors
219 # sudo dnf group install --assumeyes 'Development Tools'
220
221 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
222}
223
224wedge-deps-alpine() {
225 # https://linuxconfig.org/install-development-tools-on-redhat-8
226 # Trying to get past compile errors
227 # sudo dnf group install --assumeyes 'Development Tools'
228
229 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
230}
231
232#
233# Unused patch, was experiment for Fedora
234#
235
236get-typed-ast-patch() {
237 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
238}
239
240# Work around typed_ast bug:
241# https://github.com/python/typed_ast/issues/169
242#
243# Apply this patch
244# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
245#
246# typed_ast is tarred up though
247patch-typed-ast() {
248 local package_dir=_cache/py3-libs
249 local patch=$PWD/deps/typed_ast.patch
250
251 pushd $package_dir
252 cat $patch
253 echo
254
255 local dir=typed_ast-1.4.3
256 local tar=typed_ast-1.4.3.tar.gz
257
258 echo OLD
259 ls -l $tar
260 echo
261
262 rm -r -f -v $dir
263 tar -x -z < $tar
264
265 pushd $dir
266 patch -p1 < $patch
267 popd
268 #find $dir
269
270 # Create a new one
271 tar --create --gzip --file $tar typed_ast-1.4.3
272
273 echo NEW
274 ls -l $tar
275 echo
276
277 popd
278}
279
280#
281# Fetch
282#
283
284download-to() {
285 local dir=$1
286 local url=$2
287 wget --no-clobber --directory-prefix "$dir" "$url"
288}
289
290maybe-extract() {
291 local wedge_dir=$1
292 local tar_name=$2
293 local out_dir=$3
294
295 if test -d "$wedge_dir/$out_dir"; then
296 log "Not extracting because $wedge_dir/$out_dir exists"
297 return
298 fi
299
300 local tar=$wedge_dir/$tar_name
301 case $tar_name in
302 *.gz|*.tgz) # mksh ends with .tgz
303 flag='--gzip'
304 ;;
305 *.bz2)
306 flag='--bzip2'
307 ;;
308 *.xz)
309 flag='--xz'
310 ;;
311 *)
312 die "tar with unknown extension: $tar_name"
313 ;;
314 esac
315
316 tar --extract $flag --file $tar --directory $wedge_dir
317}
318
319clone-mypy() {
320 ### replaces deps/from-git
321 local dest_dir=$1
322 local version=${2:-$MYPY_VERSION}
323
324 local dest=$dest_dir/mypy-$version
325 if test -d $dest; then
326 log "Not cloning because $dest exists"
327 return
328 fi
329
330 # v$VERSION is a tag, not a branch
331
332 # size optimization: --depth=1 --shallow-submodules
333 # https://git-scm.com/docs/git-clone
334
335 git clone --recursive --branch v$version \
336 --depth=1 --shallow-submodules \
337 $MYPY_GIT_URL $dest
338
339 # TODO: verify commit checksum
340}
341
342copy-source-medo() {
343 mkdir -p $DEPS_SOURCE_DIR
344
345 # Copy the whole tree, including the .treeptr files
346 cp --verbose --recursive --no-target-directory \
347 deps/source.medo/ $DEPS_SOURCE_DIR/
348}
349
350fetch() {
351 local py_only=${1:-}
352
353 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
354 # would do: fetch compressed tarballs designated by .treeptr files, and
355 # expand them.
356
357 # _build/deps-source/
358 # re2c/
359 # WEDGE
360 # re2c-3.0/ # expanded .tar.xz file
361
362 copy-source-medo
363
364 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
365 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
366 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
367 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
368
369 if test -n "$py_only"; then
370 log "Fetched dependencies for 'build/py.sh'"
371 return
372 fi
373
374 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
375 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
376 pyflakes-$PYFLAKES_VERSION
377
378 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
379 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
380 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
381 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
382
383 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
384 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" dash-$BASH_VER
385
386 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
387 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
388
389 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
390 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
391
392 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
393 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
394
395 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
396 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
397
398 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
399 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$DASH_VERSION
400
401 # Patch: this tarball doesn't follow the convention $name-$version
402 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
403 pushd $DEPS_SOURCE_DIR/mksh
404 mv -v mksh mksh-$MKSH_VERSION
405 popd
406 fi
407
408 # bloaty and uftrace are for benchmarks, in containers
409 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
410 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
411 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
412 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
413
414 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
415
416 # It's also copied into a wedge in install-wedges.
417 clone-mypy $DEPS_SOURCE_DIR/mypy
418
419 if command -v tree > /dev/null; then
420 tree -L 2 $DEPS_SOURCE_DIR
421 fi
422}
423
424fetch-py() {
425 fetch py_only
426}
427
428mirror-pyflakes() {
429 ### Workaround for network error during release
430 scp \
431 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
432 oilshell.org:oilshell.org/blob/
433}
434
435wedge-exists() {
436 ### Does an installed wedge already exist?
437
438 local name=$1
439 local version=$2
440 local wedge_dir=${3:-/wedge/oils-for-unix.org}
441
442 local installed=$wedge_dir/pkg/$name/$version
443
444 if test -d $installed; then
445 log "$installed already exists"
446 return 0
447 else
448 return 1
449 fi
450}
451
452#
453# Install
454#
455
456# TODO: py3-libs needs to be a WEDGE, so that that you can run
457# 'wedge build deps/source.medo/py3-libs/' and then get it in
458#
459# _build/wedge/{absolute,relative} # which one?
460#
461# It needs a BUILD DEPENDENCY on:
462# - the python3 wedge, so you can do python3 -m pip install.
463# - the mypy repo, which has test-requirements.txt
464
465download-py3-libs() {
466 ### Download source/binary packages, AFTER python3 is installed
467
468 # Note that this is NOT source code; there is binary code, e.g. in
469 # lxml-*.whl
470
471 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
472 local py_package_dir=_cache/py3-libs
473 mkdir -p $py_package_dir
474
475 # Avoids a warning, but doesn't fix typed_ast
476 #python3 -m pip download -d $py_package_dir wheel
477
478 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
479 python3 -m pip download -d $py_package_dir pexpect
480}
481
482install-py3-libs-in-venv() {
483 local venv_dir=$1
484 local mypy_dir=$2 # This is a param for host build vs. container build
485 local package_dir=_cache/py3-libs
486
487 source $venv_dir/bin/activate # enter virtualenv
488
489 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
490 # conflicts with MyPy deps!
491 # "ERROR: pip's dependency resolver does not currently take into account all
492 # the packages that are installed."
493
494 # --find-links uses a "cache dir" for packages (weird flag name!)
495
496 # Avoids a warning, but doesn't fix typed_ast
497 #time python3 -m pip install --find-links $package_dir wheel
498
499 # for mycpp/
500 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
501
502 # pexpect: for spec/stateful/*.py
503 time python3 -m pip install --find-links $package_dir pexpect
504}
505
506install-py3-libs-from-cache() {
507
508 # As well as end users
509
510 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
511
512 local py3
513 py3=$(command -v python3)
514 case $py3 in
515 *wedge/oils-for-unix.org/*)
516 ;;
517 *)
518 die "python3 is '$py3', but expected it to be in a wedge"
519 ;;
520 esac
521
522 log "Ensuring pip is installed (interpreter $(command -v python3)"
523 python3 -m ensurepip
524
525 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
526 log "Creating venv in $venv_dir"
527
528 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
529 # the /wedge we just built
530 python3 -m venv $venv_dir
531
532 log "Installing MyPy deps in venv"
533
534 # Run in a subshell because it mutates shell state
535 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
536}
537
538install-py3-libs() {
539 ### Invoked by Dockerfile.cpp-small, etc.
540
541 download-py3-libs
542 install-py3-libs-from-cache
543}
544
545# OBSOLETE in favor of install-spec-bin-fast
546install-spec-bin() {
547 if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
548 deps/wedge.sh unboxed-build _build/deps-source/dash
549 fi
550
551 if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
552 deps/wedge.sh unboxed-build _build/deps-source/mksh
553 fi
554
555 if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
556 deps/wedge.sh unboxed-build _build/deps-source/busybox
557 fi
558
559 # Fedora compile error - count_all_jobs
560 if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
561 deps/wedge.sh unboxed-build _build/deps-source/bash
562 fi
563
564 # Fedora compiler error
565 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
566 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
567 # like ncurses-devel should fix this, but it doesn't
568 #
569 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
570 #
571 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
572 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
573 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
574 #
575 # I think the ./configure is out of sync with the actual build?
576
577 if ! wedge-exists zsh $ZSH_VERSION ''; then
578 deps/wedge.sh unboxed-build _build/deps-source/zsh
579 fi
580
581 return
582
583 # Hm this has problem with out-of-tree build? I think Oils does too actually
584 if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
585 deps/wedge.sh unboxed-build _build/deps-source/yash
586 fi
587}
588
589# TODO:
590# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
591# already in each WEDGE file
592
593py-wedges() {
594 ### for build/py.sh all
595
596 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
597 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
598 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
599 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
600}
601
602cpp-wedges() {
603 ### for ninja / mycpp translation
604
605 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
606 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
607
608 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
609 # separately for now
610 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
611}
612
613spec-bin-wedges() {
614 ### for test/spec-py.sh osh-all
615
616 echo dash $DASH_VERSION $USER_WEDGE_DIR
617 echo bash $BASH_VER $USER_WEDGE_DIR
618 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
619 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
620 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
621}
622
623timestamp() {
624 date '+%H:%M:%S'
625}
626
627my-time-tsv() {
628 python3 benchmarks/time_.py \
629 --tsv \
630 --time-span --rusage \
631 "$@"
632}
633
634maybe-install-wedge() {
635 local name=$1
636 local version=$2
637 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
638
639 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
640 local log_file=$WEDGE_LOG_DIR/$name.log.txt
641
642 echo " TASK $(timestamp) $name $version > $log_file"
643
644 # python3 because it's OUTSIDE the container
645 # Separate columns that could be joined: number of files, total size
646 my-time-tsv --print-header \
647 --field xargs_slot \
648 --field wedge \
649 --field wedge_HREF \
650 --field version \
651 --output $task_file
652
653 if wedge-exists "$name" "$version" "$wedge_dir"; then
654 echo "CACHED $(timestamp) $name $version"
655 return
656 fi
657
658 local -a cmd=( deps/wedge.sh unboxed-build _build/deps-source/$name/ )
659
660 set +o errexit
661 my-time-tsv \
662 --field "$XARGS_SLOT" \
663 --field "$name" \
664 --field "$name.log.txt" \
665 --field "$version" \
666 --append \
667 --output $task_file \
668 "${cmd[@]}" "$@" >$log_file 2>&1
669 local status=$?
670 set -o errexit
671
672 if test "$status" -eq 0; then
673 echo " OK $(timestamp) $name $version"
674 else
675 echo " FAIL $(timestamp) $name $version"
676 fi
677}
678
679dummy-task() {
680 ### For testing log capture
681 local name=$1
682 local version=$2
683
684 echo "Building $name $version"
685
686 # random float between 0 and 3
687 # weirdly we need a seed from bash
688 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
689 local secs
690 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
691
692 echo "sleep $secs"
693 sleep $secs
694
695 echo 'stdout'
696 log 'stderr'
697
698 if test $name = 'mksh'; then
699 echo "simulate failure for $name"
700 exit 2
701 fi
702}
703
704readonly WEDGE_LOG_DIR=_build/wedge/logs
705
706dummy-task-wrapper() {
707 # Similar to test/common.sh run-task-with-status, used by
708 # test/{spec,wild}-runner.sh
709 local name=$1
710 local version=$2
711
712 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
713 local log_file=$WEDGE_LOG_DIR/$name.log.txt
714
715 echo " TASK $(timestamp) $name $version > $log_file"
716
717 # python3 because it's OUTSIDE the container
718 # Separate columns that could be joined: number of files, total size
719 my-time-tsv --print-header \
720 --field xargs_slot \
721 --field wedge \
722 --field wedge_HREF \
723 --field version \
724 --output $task_file
725
726 my-time-tsv \
727 --field "$XARGS_SLOT" \
728 --field "$name" \
729 --field "$name.log.txt" \
730 --field "$version" \
731 --append \
732 --output $task_file \
733 $0 dummy-task "$@" >$log_file 2>&1 || true
734
735 echo " DONE $(timestamp) $name $version"
736}
737
738html-head() {
739 # python3 because we're outside containers
740 PYTHONPATH=. python3 doctools/html_head.py "$@"
741}
742
743index-html() {
744 local tasks_tsv=$1
745
746 local base_url='../../../web'
747 html-head --title 'Wedge Builds' \
748 "$base_url/ajax.js" \
749 "$base_url/table/table-sort.js" \
750 "$base_url/table/table-sort.css" \
751 "$base_url/base.css"\
752
753 table-sort-begin 'width60'
754
755 cat <<EOF
756 <p id="home-link">
757 <a href="/">oilshell.org</a>
758 </p>
759
760 <h1>Wedge Builds</h1>
761EOF
762
763 tsv2html3 $tasks_tsv
764
765 cat <<EOF
766 <p>
767 <a href="tasks.tsv">tasks.tsv</a>
768 </p>
769EOF
770
771 table-sort-end 'tasks' # ID for sorting
772}
773
774NPROC=$(nproc)
775#NPROC=1
776
777install-wedge-list() {
778 ### Reads task rows from stdin
779 local parallel=${1:-}
780
781
782 mkdir -p _build/wedge/logs
783
784 local -a flags
785 if test -n "$parallel"; then
786 log ""
787 log "=== Installing wedges with $NPROC jobs in parallel"
788 log ""
789 flags=( -P $NPROC )
790 else
791 log ""
792 log "=== Installing wedges serially"
793 log ""
794 fi
795
796 # Reads from stdin
797 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
798 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
799
800 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
801}
802
803write-task-report() {
804 local tasks_tsv=_build/wedge/logs/tasks.tsv
805
806 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
807 log "Wrote $tasks_tsv"
808
809 # TODO: version can be right-justified?
810 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
811column_name type precision strftime
812status integer 0 -
813elapsed_secs float 1 -
814user_secs float 1 -
815start_time float 1 %H:%M:%S
816end_time float 1 %H:%M:%S
817sys_secs float 1 -
818max_rss_KiB integer 0 -
819xargs_slot integer 0 -
820wedge string 0 -
821wedge_HREF string 0 -
822version string 0 -
823EOF
824
825 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
826 log "Wrote $WEDGE_LOG_DIR/index.html"
827}
828
829install-spec-bin-fast() {
830 spec-bin-wedges | install-wedge-list T
831 write-task-report
832}
833
834fake-py3-libs-wedge() {
835 local name=py3-libs
836 local version=$PY3_LIBS_VERSION
837
838 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
839 local log_file=$WEDGE_LOG_DIR/$name.log.txt
840
841 my-time-tsv --print-header \
842 --field xargs_slot \
843 --field wedge \
844 --field wedge_HREF \
845 --field version \
846 --output $task_file
847
848 # There is no xargs slot!
849 my-time-tsv \
850 --field "-1" \
851 --field "$name" \
852 --field "$name.log.txt" \
853 --field "$version" \
854 --append \
855 --output $task_file \
856 $0 install-py3-libs >$log_file 2>&1 || true
857
858 echo " FAKE $(timestamp) $name $version"
859}
860
861install-wedges-fast() {
862 echo " START $(timestamp)"
863
864 # Do all of them in parallel
865 { py-wedges; cpp-wedges; spec-bin-wedges; } | install-wedge-list T
866
867 fake-py3-libs-wedge
868 echo " END $(timestamp)"
869
870 write-task-report
871}
872
873# OBSOLETE in favor of install-wedges-fast
874install-wedges() {
875 local py_only=${1:-}
876
877 # TODO:
878 # - Make all of these RELATIVE wedges
879 # - Add
880 # - unboxed-rel-smoke-test -- move it inside container
881 # - rel-smoke-test -- mount it in a different location
882
883 if ! wedge-exists cmark $CMARK_VERSION; then
884 deps/wedge.sh unboxed-build _build/deps-source/cmark/
885 fi
886
887 if ! wedge-exists re2c $RE2C_VERSION; then
888 deps/wedge.sh unboxed-build _build/deps-source/re2c/
889 fi
890
891 if ! wedge-exists python2 $PY2_VERSION; then
892 deps/wedge.sh unboxed-build _build/deps-source/python2/
893 fi
894
895 if test -n "$py_only"; then
896 log "Installed dependencies for 'build/py.sh'"
897 return
898 fi
899
900 # Just copy this source tarball
901 if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
902 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
903 mkdir -p $dest_dir
904
905 cp --verbose --recursive --no-target-directory \
906 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
907 fi
908
909 if ! wedge-exists python3 $PY3_VERSION; then
910 deps/wedge.sh unboxed-build _build/deps-source/python3/
911 fi
912
913 # Copy all the contents, except for .git folder.
914 if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
915
916 # NOTE: We have to also copy the .git dir, because it has
917 # .git/modules/typeshed
918 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
919 mkdir -p $dest_dir
920
921 # Note: pack files in .git/modules/typeshed/objects/pack are read-only
922 # this can fail
923 cp --verbose --recursive --no-target-directory \
924 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
925 fi
926
927 if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
928 download-py3-libs
929 # This patch doesn't work?
930 # patch-typed-ast
931 install-py3-libs
932 fi
933
934 if command -v tree > /dev/null; then
935 tree -L 3 $USER_WEDGE_DIR
936 echo
937 tree -L 3 /wedge/oils-for-unix.org
938 fi
939}
940
941install-wedges-py() {
942 install-wedges py_only
943}
944
945#
946# Unboxed wedge builds
947#
948
949uftrace-host() {
950 ### built on demand; run $0 first
951
952 # BUG: doesn't detect python3
953 # WEDGE tells me that it depends on pkg-config
954 # 'apt-get install pkgconf' gets it
955 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
956
957 deps/wedge.sh unboxed-build _build/deps-source/uftrace
958}
959
960bloaty-host() {
961 deps/wedge.sh unboxed-build _build/deps-source/bloaty
962}
963
964R-libs-host() {
965 deps/wedge.sh unboxed-build _build/deps-source/R-libs
966}
967
968#
969# Wedges built inside a container, for copying into a container
970#
971
972container-wedges() {
973 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
974
975 #export-podman
976
977 if true; then
978 deps/wedge.sh build deps/source.medo/time-helper
979 deps/wedge.sh build deps/source.medo/cmark/
980 deps/wedge.sh build deps/source.medo/re2c/
981 deps/wedge.sh build deps/source.medo/python3/
982 fi
983
984 if false; then
985 deps/wedge.sh build deps/source.medo/bloaty/
986 deps/wedge.sh build deps/source.medo/uftrace/
987 fi
988
989 if false; then
990 # For soil-benchmarks/ images
991 deps/wedge.sh build deps/source.medo/R-libs/
992 fi
993}
994
995#
996# Report
997#
998
999commas() {
1000 # Wow I didn't know this :a trick
1001 #
1002 # OK this is a label and a loop, which makes sense. You can't do it with
1003 # pure regex.
1004 #
1005 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1006 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1007 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1008}
1009
1010wedge-sizes() {
1011 local tmp=_tmp/wedge-sizes.txt
1012
1013 # -b is --bytes, but use short flag for busybox compat
1014 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1015 { print $0 # print the line
1016 total_bytes += $1 # accumulate
1017 }
1018END { print total_bytes " TOTAL" }
1019' > $tmp
1020
1021 # printf justifies du output
1022 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1023 echo
1024
1025 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1026 #echo
1027}
1028
1029wedge-report() {
1030 # 4 levels deep shows the package
1031 if command -v tree > /dev/null; then
1032 tree -L 4 /wedge ~/wedge
1033 echo
1034 fi
1035
1036 wedge-sizes
1037
1038 local tmp=_tmp/wedge-manifest.txt
1039
1040 echo 'Biggest files'
1041 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1042 # busybox find doesn't have -printf
1043 echo 'find -printf failed'
1044 return
1045 fi
1046
1047 set +o errexit # ignore SIGPIPE
1048 sort -n --reverse $tmp | head -n 20 | commas
1049 set -o errexit
1050
1051 echo
1052
1053 # Show the most common file extensions
1054 #
1055 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1056 # most common
1057 #
1058 # There are also duplicate .a files for Python -- should look at how distros
1059 # get rid of those
1060
1061 cat $tmp | python3 -c '
1062import os, sys, collections
1063
1064bytes = collections.Counter()
1065files = collections.Counter()
1066
1067for line in sys.stdin:
1068 size, path = line.split(None, 1)
1069 path = path.strip() # remove newline
1070 _, ext = os.path.splitext(path)
1071 size = int(size)
1072
1073 bytes[ext] += size
1074 files[ext] += 1
1075
1076#print(bytes)
1077#print(files)
1078
1079n = 20
1080
1081print("Most common file types")
1082for ext, count in files.most_common()[:n]:
1083 print("%10d %s" % (count, ext))
1084
1085print()
1086
1087print("Total bytes by file type")
1088for ext, total_bytes in bytes.most_common()[:n]:
1089 print("%10d %s" % (total_bytes, ext))
1090' | commas
1091}
1092
1093run-task "$@"