OILS / build / deps.sh View on Github | oilshell.org

1091 lines, 571 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
13#
14# TODO: Do we need something faster, just python2, re2c, and cmark?
15#
16# - build/deps.sh fetch-py
17# - build/deps.sh install-wedges-py
18#
19# TODO: Can we make most of them non-root deps? This requires rebuilding
20# containers, which requires podman.
21#
22# rm -r -f ~/wedge # would be better
23
24set -o nounset
25set -o pipefail
26set -o errexit
27
28REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
29
30source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
31source deps/from-apt.sh # PY3_BUILD_DEPS
32#source deps/podman.sh
33source devtools/run-task.sh # run-task
34source test/tsv-lib.sh # tsv-concat
35source web/table/html.sh # table-sort-{begin,end}
36
37# Also in build/dev-shell.sh
38USER_WEDGE_DIR=~/wedge/oils-for-unix.org
39ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
40
41readonly DEPS_SOURCE_DIR=_build/deps-source
42
43readonly RE2C_VERSION=3.0
44readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
45
46readonly CMARK_VERSION=0.29.0
47readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
48
49readonly PY2_VERSION=2.7.18
50readonly PY2_URL="https://www.python.org/ftp/python/2.7.18/Python-$PY2_VERSION.tar.xz"
51
52readonly PY3_VERSION=3.10.4
53readonly PY3_URL="https://www.python.org/ftp/python/3.10.4/Python-$PY3_VERSION.tar.xz"
54
55readonly BASH_VER=4.4 # don't clobber BASH_VERSION
56readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
57
58readonly DASH_VERSION=0.5.10.2
59readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
60
61readonly ZSH_VERSION=5.1.1
62readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
63
64readonly MKSH_VERSION=R52c
65readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
66
67readonly BUSYBOX_VERSION='1.35.0'
68readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
69
70readonly YASH_VERSION=2.49
71readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
72
73readonly MYPY_GIT_URL=https://github.com/python/mypy
74readonly MYPY_VERSION=0.780
75
76readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
77
78# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
79# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
80readonly PYFLAKES_VERSION=2.4.0
81#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
82# 2023-07: Mirrored to avoid network problem on broome during release
83readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
84
85readonly BLOATY_VERSION=1.1
86readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
87
88readonly UFTRACE_VERSION=0.13
89readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
90
91log() {
92 echo "$0: $@" >& 2
93}
94
95die() {
96 log "$@"
97 exit 1
98}
99
100rm-oils-crap() {
101 ### When you want to start over
102
103 rm -r -f -v ~/wedge
104 sudo rm -r -f -v /wedge
105}
106
107# Note: git is an implicit dependency -- that's how we got the repo in the
108# first place!
109
110# python2-dev is no longer available on Debian 12
111# python-dev also seems gone
112#
113# wget: for fetching wedges (not on Debian by default!)
114# tree: tiny package that's useful for showing what we installed
115# g++: essential
116# libreadline-dev: needed for the build/prepare.sh Python build.
117# gawk: used by spec-runner.sh for the special match() function.
118# cmake: for cmark
119# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
120# as well
121readonly -a WEDGE_DEPS_DEBIAN=(
122 wget tree gawk
123 g++ ninja-build cmake
124 libreadline-dev
125 "${PY3_BUILD_DEPS[@]}"
126)
127
128readonly -a WEDGE_DEPS_ALPINE=(
129 bzip2
130 xz
131
132 wget tree gawk
133
134 gcc g++
135 ninja-build
136 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
137 ninja-is-really-ninja
138 cmake
139
140 readline-dev
141 zlib-dev
142 libffi-dev
143 openssl-dev
144
145 ncurses-dev
146)
147
148readonly -a WEDGE_DEPS_FEDORA=(
149
150 # Weird, Fedora doesn't have these by default!
151 hostname
152 tar
153 bzip2
154
155 # https://packages.fedoraproject.org/pkgs/wget/wget/
156 wget
157 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
158 tree
159 gawk
160
161 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
162 gcc gcc-c++
163
164 ninja-build
165 cmake
166
167 readline-devel
168
169 # Like PY3_BUILD_DEPS
170 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
171 zlib-devel
172 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
173 libffi-devel
174 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
175 openssl-devel
176
177 # For building zsh from source?
178 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
179 ncurses-devel
180 #libcap-devel
181
182 # still have a job control error compiling bash
183 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
184 # glibc-devel
185)
186
187install-debian-packages() {
188 ### Packages for build/py.sh all, building wedges, etc.
189
190 set -x # show what needs sudo
191
192 # pass -y for say gitpod
193 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
194 set +x
195
196 # maybe pass -y through
197 test/spec-bin.sh install-shells-with-apt "$@"
198}
199
200install-ubuntu-packages() {
201 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
202 install-debian-packages "$@"
203}
204
205wedge-deps-debian() {
206 # Install packages without prompt
207
208 # 2024-02 - there was an Ubuntu update, and we started needing this
209 sudo apt-get -y update
210
211 install-debian-packages -y
212}
213
214wedge-deps-fedora() {
215 # https://linuxconfig.org/install-development-tools-on-redhat-8
216 # Trying to get past compile errors
217 # sudo dnf group install --assumeyes 'Development Tools'
218
219 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
220}
221
222wedge-deps-alpine() {
223 # https://linuxconfig.org/install-development-tools-on-redhat-8
224 # Trying to get past compile errors
225 # sudo dnf group install --assumeyes 'Development Tools'
226
227 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
228}
229
230#
231# Unused patch, was experiment for Fedora
232#
233
234get-typed-ast-patch() {
235 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
236}
237
238# Work around typed_ast bug:
239# https://github.com/python/typed_ast/issues/169
240#
241# Apply this patch
242# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
243#
244# typed_ast is tarred up though
245patch-typed-ast() {
246 local package_dir=_cache/py3-libs
247 local patch=$PWD/deps/typed_ast.patch
248
249 pushd $package_dir
250 cat $patch
251 echo
252
253 local dir=typed_ast-1.4.3
254 local tar=typed_ast-1.4.3.tar.gz
255
256 echo OLD
257 ls -l $tar
258 echo
259
260 rm -r -f -v $dir
261 tar -x -z < $tar
262
263 pushd $dir
264 patch -p1 < $patch
265 popd
266 #find $dir
267
268 # Create a new one
269 tar --create --gzip --file $tar typed_ast-1.4.3
270
271 echo NEW
272 ls -l $tar
273 echo
274
275 popd
276}
277
278#
279# Fetch
280#
281
282download-to() {
283 local dir=$1
284 local url=$2
285 wget --no-clobber --directory-prefix "$dir" "$url"
286}
287
288maybe-extract() {
289 local wedge_dir=$1
290 local tar_name=$2
291 local out_dir=$3
292
293 if test -d "$wedge_dir/$out_dir"; then
294 log "Not extracting because $wedge_dir/$out_dir exists"
295 return
296 fi
297
298 local tar=$wedge_dir/$tar_name
299 case $tar_name in
300 *.gz|*.tgz) # mksh ends with .tgz
301 flag='--gzip'
302 ;;
303 *.bz2)
304 flag='--bzip2'
305 ;;
306 *.xz)
307 flag='--xz'
308 ;;
309 *)
310 die "tar with unknown extension: $tar_name"
311 ;;
312 esac
313
314 tar --extract $flag --file $tar --directory $wedge_dir
315}
316
317clone-mypy() {
318 ### replaces deps/from-git
319 local dest_dir=$1
320 local version=${2:-$MYPY_VERSION}
321
322 local dest=$dest_dir/mypy-$version
323 if test -d $dest; then
324 log "Not cloning because $dest exists"
325 return
326 fi
327
328 # v$VERSION is a tag, not a branch
329
330 # size optimization: --depth=1 --shallow-submodules
331 # https://git-scm.com/docs/git-clone
332
333 git clone --recursive --branch v$version \
334 --depth=1 --shallow-submodules \
335 $MYPY_GIT_URL $dest
336
337 # TODO: verify commit checksum
338}
339
340copy-source-medo() {
341 mkdir -p $DEPS_SOURCE_DIR
342
343 # Copy the whole tree, including the .treeptr files
344 cp --verbose --recursive --no-target-directory \
345 deps/source.medo/ $DEPS_SOURCE_DIR/
346}
347
348fetch() {
349 local py_only=${1:-}
350
351 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
352 # would do: fetch compressed tarballs designated by .treeptr files, and
353 # expand them.
354
355 # _build/deps-source/
356 # re2c/
357 # WEDGE
358 # re2c-3.0/ # expanded .tar.xz file
359
360 copy-source-medo
361
362 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
363 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
364 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
365 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
366
367 if test -n "$py_only"; then
368 log "Fetched dependencies for 'build/py.sh'"
369 return
370 fi
371
372 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
373 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
374 pyflakes-$PYFLAKES_VERSION
375
376 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
377 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
378 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
379 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
380
381 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
382 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" dash-$BASH_VER
383
384 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
385 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
386
387 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
388 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
389
390 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
391 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
392
393 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
394 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
395
396 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
397 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$DASH_VERSION
398
399 # Patch: this tarball doesn't follow the convention $name-$version
400 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
401 pushd $DEPS_SOURCE_DIR/mksh
402 mv -v mksh mksh-$MKSH_VERSION
403 popd
404 fi
405
406 # bloaty and uftrace are for benchmarks, in containers
407 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
408 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
409 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
410 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
411
412 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
413
414 # It's also copied into a wedge in install-wedges.
415 clone-mypy $DEPS_SOURCE_DIR/mypy
416
417 if command -v tree > /dev/null; then
418 tree -L 2 $DEPS_SOURCE_DIR
419 fi
420}
421
422fetch-py() {
423 fetch py_only
424}
425
426mirror-pyflakes() {
427 ### Workaround for network error during release
428 scp \
429 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
430 oilshell.org:oilshell.org/blob/
431}
432
433wedge-exists() {
434 ### Does an installed wedge already exist?
435
436 local name=$1
437 local version=$2
438 local wedge_dir=${3:-/wedge/oils-for-unix.org}
439
440 local installed=$wedge_dir/pkg/$name/$version
441
442 if test -d $installed; then
443 log "$installed already exists"
444 return 0
445 else
446 return 1
447 fi
448}
449
450#
451# Install
452#
453
454# TODO: py3-libs needs to be a WEDGE, so that that you can run
455# 'wedge build deps/source.medo/py3-libs/' and then get it in
456#
457# _build/wedge/{absolute,relative} # which one?
458#
459# It needs a BUILD DEPENDENCY on:
460# - the python3 wedge, so you can do python3 -m pip install.
461# - the mypy repo, which has test-requirements.txt
462
463download-py3-libs() {
464 ### Download source/binary packages, AFTER python3 is installed
465
466 # Note that this is NOT source code; there is binary code, e.g. in
467 # lxml-*.whl
468
469 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
470 local py_package_dir=_cache/py3-libs
471 mkdir -p $py_package_dir
472
473 # Avoids a warning, but doesn't fix typed_ast
474 #python3 -m pip download -d $py_package_dir wheel
475
476 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
477 python3 -m pip download -d $py_package_dir pexpect
478}
479
480install-py3-libs-in-venv() {
481 local venv_dir=$1
482 local mypy_dir=$2 # This is a param for host build vs. container build
483 local package_dir=_cache/py3-libs
484
485 source $venv_dir/bin/activate # enter virtualenv
486
487 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
488 # conflicts with MyPy deps!
489 # "ERROR: pip's dependency resolver does not currently take into account all
490 # the packages that are installed."
491
492 # --find-links uses a "cache dir" for packages (weird flag name!)
493
494 # Avoids a warning, but doesn't fix typed_ast
495 #time python3 -m pip install --find-links $package_dir wheel
496
497 # for mycpp/
498 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
499
500 # pexpect: for spec/stateful/*.py
501 time python3 -m pip install --find-links $package_dir pexpect
502}
503
504install-py3-libs-from-cache() {
505
506 # As well as end users
507
508 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
509
510 local py3
511 py3=$(command -v python3)
512 case $py3 in
513 *wedge/oils-for-unix.org/*)
514 ;;
515 *)
516 die "python3 is '$py3', but expected it to be in a wedge"
517 ;;
518 esac
519
520 log "Ensuring pip is installed (interpreter $(command -v python3)"
521 python3 -m ensurepip
522
523 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
524 log "Creating venv in $venv_dir"
525
526 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
527 # the /wedge we just built
528 python3 -m venv $venv_dir
529
530 log "Installing MyPy deps in venv"
531
532 # Run in a subshell because it mutates shell state
533 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
534}
535
536install-py3-libs() {
537 ### Invoked by Dockerfile.cpp-small, etc.
538
539 download-py3-libs
540 install-py3-libs-from-cache
541}
542
543# OBSOLETE in favor of install-spec-bin-fast
544install-spec-bin() {
545 if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
546 deps/wedge.sh unboxed-build _build/deps-source/dash
547 fi
548
549 if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
550 deps/wedge.sh unboxed-build _build/deps-source/mksh
551 fi
552
553 if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
554 deps/wedge.sh unboxed-build _build/deps-source/busybox
555 fi
556
557 # Fedora compile error - count_all_jobs
558 if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
559 deps/wedge.sh unboxed-build _build/deps-source/bash
560 fi
561
562 # Fedora compiler error
563 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
564 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
565 # like ncurses-devel should fix this, but it doesn't
566 #
567 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
568 #
569 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
570 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
571 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
572 #
573 # I think the ./configure is out of sync with the actual build?
574
575 if ! wedge-exists zsh $ZSH_VERSION ''; then
576 deps/wedge.sh unboxed-build _build/deps-source/zsh
577 fi
578
579 return
580
581 # Hm this has problem with out-of-tree build? I think Oils does too actually
582 if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
583 deps/wedge.sh unboxed-build _build/deps-source/yash
584 fi
585}
586
587# TODO:
588# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
589# already in each WEDGE file
590
591py-wedges() {
592 ### for build/py.sh all
593
594 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
595 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
596 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
597 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
598}
599
600cpp-wedges() {
601 ### for ninja / mycpp translation
602
603 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
604 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
605
606 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
607 # separately for now
608 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
609}
610
611spec-bin-wedges() {
612 ### for test/spec-py.sh osh-all
613
614 echo dash $DASH_VERSION $USER_WEDGE_DIR
615 echo bash $BASH_VER $USER_WEDGE_DIR
616 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
617 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
618 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
619}
620
621timestamp() {
622 date '+%H:%M:%S'
623}
624
625my-time-tsv() {
626 python3 benchmarks/time_.py \
627 --tsv \
628 --time-span --rusage \
629 "$@"
630}
631
632maybe-install-wedge() {
633 local name=$1
634 local version=$2
635 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
636
637 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
638 local log_file=$WEDGE_LOG_DIR/$name.log.txt
639
640 echo " TASK $(timestamp) $name $version > $log_file"
641
642 # python3 because it's OUTSIDE the container
643 # Separate columns that could be joined: number of files, total size
644 my-time-tsv --print-header \
645 --field xargs_slot \
646 --field wedge \
647 --field wedge_HREF \
648 --field version \
649 --output $task_file
650
651 if wedge-exists "$name" "$version" "$wedge_dir"; then
652 echo "CACHED $(timestamp) $name $version"
653 return
654 fi
655
656 local -a cmd=( deps/wedge.sh unboxed-build _build/deps-source/$name/ )
657
658 set +o errexit
659 my-time-tsv \
660 --field "$XARGS_SLOT" \
661 --field "$name" \
662 --field "$name.log.txt" \
663 --field "$version" \
664 --append \
665 --output $task_file \
666 "${cmd[@]}" "$@" >$log_file 2>&1
667 local status=$?
668 set -o errexit
669
670 if test "$status" -eq 0; then
671 echo " OK $(timestamp) $name $version"
672 else
673 echo " FAIL $(timestamp) $name $version"
674 fi
675}
676
677dummy-task() {
678 ### For testing log capture
679 local name=$1
680 local version=$2
681
682 echo "Building $name $version"
683
684 # random float between 0 and 3
685 # weirdly we need a seed from bash
686 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
687 local secs
688 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
689
690 echo "sleep $secs"
691 sleep $secs
692
693 echo 'stdout'
694 log 'stderr'
695
696 if test $name = 'mksh'; then
697 echo "simulate failure for $name"
698 exit 2
699 fi
700}
701
702readonly WEDGE_LOG_DIR=_build/wedge/logs
703
704dummy-task-wrapper() {
705 # Similar to test/common.sh run-task-with-status, used by
706 # test/{spec,wild}-runner.sh
707 local name=$1
708 local version=$2
709
710 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
711 local log_file=$WEDGE_LOG_DIR/$name.log.txt
712
713 echo " TASK $(timestamp) $name $version > $log_file"
714
715 # python3 because it's OUTSIDE the container
716 # Separate columns that could be joined: number of files, total size
717 my-time-tsv --print-header \
718 --field xargs_slot \
719 --field wedge \
720 --field wedge_HREF \
721 --field version \
722 --output $task_file
723
724 my-time-tsv \
725 --field "$XARGS_SLOT" \
726 --field "$name" \
727 --field "$name.log.txt" \
728 --field "$version" \
729 --append \
730 --output $task_file \
731 $0 dummy-task "$@" >$log_file 2>&1 || true
732
733 echo " DONE $(timestamp) $name $version"
734}
735
736html-head() {
737 # python3 because we're outside containers
738 PYTHONPATH=. python3 doctools/html_head.py "$@"
739}
740
741index-html() {
742 local tasks_tsv=$1
743
744 local base_url='../../../web'
745 html-head --title 'Wedge Builds' \
746 "$base_url/ajax.js" \
747 "$base_url/table/table-sort.js" \
748 "$base_url/table/table-sort.css" \
749 "$base_url/base.css"\
750
751 table-sort-begin 'width60'
752
753 cat <<EOF
754 <p id="home-link">
755 <a href="/">oilshell.org</a>
756 </p>
757
758 <h1>Wedge Builds</h1>
759EOF
760
761 tsv2html3 $tasks_tsv
762
763 cat <<EOF
764 <p>
765 <a href="tasks.tsv">tasks.tsv</a>
766 </p>
767EOF
768
769 table-sort-end 'tasks' # ID for sorting
770}
771
772NPROC=$(nproc)
773#NPROC=1
774
775install-wedge-list() {
776 ### Reads task rows from stdin
777 local parallel=${1:-}
778
779
780 mkdir -p _build/wedge/logs
781
782 local -a flags
783 if test -n "$parallel"; then
784 log ""
785 log "=== Installing wedges with $NPROC jobs in parallel"
786 log ""
787 flags=( -P $NPROC )
788 else
789 log ""
790 log "=== Installing wedges serially"
791 log ""
792 fi
793
794 # Reads from stdin
795 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
796 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
797
798 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
799}
800
801write-task-report() {
802 local tasks_tsv=_build/wedge/logs/tasks.tsv
803
804 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
805 log "Wrote $tasks_tsv"
806
807 # TODO: version can be right-justified?
808 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
809column_name type precision strftime
810status integer 0 -
811elapsed_secs float 1 -
812user_secs float 1 -
813start_time float 1 %H:%M:%S
814end_time float 1 %H:%M:%S
815sys_secs float 1 -
816max_rss_KiB integer 0 -
817xargs_slot integer 0 -
818wedge string 0 -
819wedge_HREF string 0 -
820version string 0 -
821EOF
822
823 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
824 log "Wrote $WEDGE_LOG_DIR/index.html"
825}
826
827install-spec-bin-fast() {
828 spec-bin-wedges | install-wedge-list T
829 write-task-report
830}
831
832fake-py3-libs-wedge() {
833 local name=py3-libs
834 local version=$PY3_LIBS_VERSION
835
836 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
837 local log_file=$WEDGE_LOG_DIR/$name.log.txt
838
839 my-time-tsv --print-header \
840 --field xargs_slot \
841 --field wedge \
842 --field wedge_HREF \
843 --field version \
844 --output $task_file
845
846 # There is no xargs slot!
847 my-time-tsv \
848 --field "-1" \
849 --field "$name" \
850 --field "$name.log.txt" \
851 --field "$version" \
852 --append \
853 --output $task_file \
854 $0 install-py3-libs >$log_file 2>&1 || true
855
856 echo " FAKE $(timestamp) $name $version"
857}
858
859install-wedges-fast() {
860 echo " START $(timestamp)"
861
862 # Do all of them in parallel
863 { py-wedges; cpp-wedges; spec-bin-wedges; } | install-wedge-list T
864
865 fake-py3-libs-wedge
866 echo " END $(timestamp)"
867
868 write-task-report
869}
870
871# OBSOLETE in favor of install-wedges-fast
872install-wedges() {
873 local py_only=${1:-}
874
875 # TODO:
876 # - Make all of these RELATIVE wedges
877 # - Add
878 # - unboxed-rel-smoke-test -- move it inside container
879 # - rel-smoke-test -- mount it in a different location
880
881 if ! wedge-exists cmark $CMARK_VERSION; then
882 deps/wedge.sh unboxed-build _build/deps-source/cmark/
883 fi
884
885 if ! wedge-exists re2c $RE2C_VERSION; then
886 deps/wedge.sh unboxed-build _build/deps-source/re2c/
887 fi
888
889 if ! wedge-exists python2 $PY2_VERSION; then
890 deps/wedge.sh unboxed-build _build/deps-source/python2/
891 fi
892
893 if test -n "$py_only"; then
894 log "Installed dependencies for 'build/py.sh'"
895 return
896 fi
897
898 # Just copy this source tarball
899 if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
900 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
901 mkdir -p $dest_dir
902
903 cp --verbose --recursive --no-target-directory \
904 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
905 fi
906
907 if ! wedge-exists python3 $PY3_VERSION; then
908 deps/wedge.sh unboxed-build _build/deps-source/python3/
909 fi
910
911 # Copy all the contents, except for .git folder.
912 if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
913
914 # NOTE: We have to also copy the .git dir, because it has
915 # .git/modules/typeshed
916 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
917 mkdir -p $dest_dir
918
919 # Note: pack files in .git/modules/typeshed/objects/pack are read-only
920 # this can fail
921 cp --verbose --recursive --no-target-directory \
922 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
923 fi
924
925 if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
926 download-py3-libs
927 # This patch doesn't work?
928 # patch-typed-ast
929 install-py3-libs
930 fi
931
932 if command -v tree > /dev/null; then
933 tree -L 3 $USER_WEDGE_DIR
934 echo
935 tree -L 3 /wedge/oils-for-unix.org
936 fi
937}
938
939install-wedges-py() {
940 install-wedges py_only
941}
942
943#
944# Unboxed wedge builds
945#
946
947uftrace-host() {
948 ### built on demand; run $0 first
949
950 # BUG: doesn't detect python3
951 # WEDGE tells me that it depends on pkg-config
952 # 'apt-get install pkgconf' gets it
953 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
954
955 deps/wedge.sh unboxed-build _build/deps-source/uftrace
956}
957
958bloaty-host() {
959 deps/wedge.sh unboxed-build _build/deps-source/bloaty
960}
961
962R-libs-host() {
963 deps/wedge.sh unboxed-build _build/deps-source/R-libs
964}
965
966#
967# Wedges built inside a container, for copying into a container
968#
969
970container-wedges() {
971 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
972
973 #export-podman
974
975 if true; then
976 deps/wedge.sh build deps/source.medo/time-helper
977 deps/wedge.sh build deps/source.medo/cmark/
978 deps/wedge.sh build deps/source.medo/re2c/
979 deps/wedge.sh build deps/source.medo/python3/
980 fi
981
982 if false; then
983 deps/wedge.sh build deps/source.medo/bloaty/
984 deps/wedge.sh build deps/source.medo/uftrace/
985 fi
986
987 if false; then
988 # For soil-benchmarks/ images
989 deps/wedge.sh build deps/source.medo/R-libs/
990 fi
991}
992
993#
994# Report
995#
996
997commas() {
998 # Wow I didn't know this :a trick
999 #
1000 # OK this is a label and a loop, which makes sense. You can't do it with
1001 # pure regex.
1002 #
1003 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1004 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1005 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1006}
1007
1008wedge-sizes() {
1009 local tmp=_tmp/wedge-sizes.txt
1010
1011 # -b is --bytes, but use short flag for busybox compat
1012 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1013 { print $0 # print the line
1014 total_bytes += $1 # accumulate
1015 }
1016END { print total_bytes " TOTAL" }
1017' > $tmp
1018
1019 # printf justifies du output
1020 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1021 echo
1022
1023 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1024 #echo
1025}
1026
1027wedge-report() {
1028 # 4 levels deep shows the package
1029 if command -v tree > /dev/null; then
1030 tree -L 4 /wedge ~/wedge
1031 echo
1032 fi
1033
1034 wedge-sizes
1035
1036 local tmp=_tmp/wedge-manifest.txt
1037
1038 echo 'Biggest files'
1039 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1040 # busybox find doesn't have -printf
1041 echo 'find -printf failed'
1042 return
1043 fi
1044
1045 set +o errexit # ignore SIGPIPE
1046 sort -n --reverse $tmp | head -n 20 | commas
1047 set -o errexit
1048
1049 echo
1050
1051 # Show the most common file extensions
1052 #
1053 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1054 # most common
1055 #
1056 # There are also duplicate .a files for Python -- should look at how distros
1057 # get rid of those
1058
1059 cat $tmp | python3 -c '
1060import os, sys, collections
1061
1062bytes = collections.Counter()
1063files = collections.Counter()
1064
1065for line in sys.stdin:
1066 size, path = line.split(None, 1)
1067 path = path.strip() # remove newline
1068 _, ext = os.path.splitext(path)
1069 size = int(size)
1070
1071 bytes[ext] += size
1072 files[ext] += 1
1073
1074#print(bytes)
1075#print(files)
1076
1077n = 20
1078
1079print("Most common file types")
1080for ext, count in files.most_common()[:n]:
1081 print("%10d %s" % (count, ext))
1082
1083print()
1084
1085print("Total bytes by file type")
1086for ext, total_bytes in bytes.most_common()[:n]:
1087 print("%10d %s" % (total_bytes, ext))
1088' | commas
1089}
1090
1091run-task "$@"