OILS / build / deps.sh View on Github | oilshell.org

1119 lines, 590 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
13#
14# TODO: Do we need something faster, just python2, re2c, and cmark?
15#
16# - build/deps.sh fetch-py
17# - build/deps.sh install-wedges-py
18#
19# TODO: Can we make most of them non-root deps? This requires rebuilding
20# containers, which requires podman.
21#
22# rm -r -f ~/wedge # would be better
23
24set -o nounset
25set -o pipefail
26set -o errexit
27
28REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
29
30source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
31source deps/from-apt.sh # PY3_BUILD_DEPS
32#source deps/podman.sh
33source devtools/run-task.sh # run-task
34source test/tsv-lib.sh # tsv-concat
35source web/table/html.sh # table-sort-{begin,end}
36
37# Also in build/dev-shell.sh
38USER_WEDGE_DIR=~/wedge/oils-for-unix.org
39ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
40
41readonly DEPS_SOURCE_DIR=_build/deps-source
42
43readonly RE2C_VERSION=3.0
44readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
45
46readonly CMARK_VERSION=0.29.0
47readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
48
49readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
50
51readonly PY2_VERSION=2.7.18
52readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
53
54readonly PY3_VERSION=3.10.4
55readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
56
57readonly BASH_VER=4.4 # don't clobber BASH_VERSION
58readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
59
60readonly DASH_VERSION=0.5.10.2
61readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
62
63readonly ZSH_VERSION=5.1.1
64readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
65
66readonly MKSH_VERSION=R52c
67readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
68
69readonly BUSYBOX_VERSION='1.35.0'
70readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
71
72readonly YASH_VERSION=2.49
73readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
74
75readonly MYPY_GIT_URL=https://github.com/python/mypy
76readonly MYPY_VERSION=0.780
77
78readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
79
80# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
81# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
82readonly PYFLAKES_VERSION=2.4.0
83#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
84# 2023-07: Mirrored to avoid network problem on broome during release
85readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
86
87readonly BLOATY_VERSION=1.1
88readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
89
90readonly UFTRACE_VERSION=0.13
91readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
92
93readonly SOUFFLE_VERSION=2.4.1
94readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
95
96log() {
97 echo "$0: $@" >& 2
98}
99
100die() {
101 log "$@"
102 exit 1
103}
104
105rm-oils-crap() {
106 ### When you want to start over
107
108 rm -r -f -v ~/wedge
109 sudo rm -r -f -v /wedge
110}
111
112# Note: git is an implicit dependency -- that's how we got the repo in the
113# first place!
114
115# python2-dev is no longer available on Debian 12
116# python-dev also seems gone
117#
118# wget: for fetching wedges (not on Debian by default!)
119# tree: tiny package that's useful for showing what we installed
120# g++: essential
121# libreadline-dev: needed for the build/prepare.sh Python build.
122# gawk: used by spec-runner.sh for the special match() function.
123# cmake: for cmark
124# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
125# as well
126readonly -a WEDGE_DEPS_DEBIAN=(
127 bzip2
128 wget
129 tree
130 gawk
131 g++
132 ninja-build
133 cmake
134 libreadline-dev
135 systemtap-sdt-dev
136
137 # for Souffle, flex and bison
138 flex
139 bison
140
141 "${PY3_BUILD_DEPS[@]}"
142)
143
144readonly -a WEDGE_DEPS_ALPINE=(
145 bzip2
146 xz
147
148 wget tree gawk
149
150 gcc g++
151 ninja-build
152 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
153 ninja-is-really-ninja
154 cmake
155
156 readline-dev
157 zlib-dev
158 libffi-dev
159 openssl-dev
160
161 ncurses-dev
162
163 # for Souffle, flex and bison
164 flex
165 bison
166)
167
168readonly -a WEDGE_DEPS_FEDORA=(
169
170 # Weird, Fedora doesn't have these by default!
171 hostname
172 tar
173 bzip2
174
175 # https://packages.fedoraproject.org/pkgs/wget/wget/
176 wget
177 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
178 tree
179 gawk
180
181 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
182 gcc gcc-c++
183
184 ninja-build
185 cmake
186
187 readline-devel
188
189 # Like PY3_BUILD_DEPS
190 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
191 zlib-devel
192 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
193 libffi-devel
194 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
195 openssl-devel
196
197 # For building zsh from source?
198 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
199 ncurses-devel
200 #libcap-devel
201
202 # still have a job control error compiling bash
203 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
204 # glibc-devel
205)
206
207install-debian-packages() {
208 ### Packages for build/py.sh all, building wedges, etc.
209
210 set -x # show what needs sudo
211
212 # pass -y for say gitpod
213 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
214 set +x
215
216 # maybe pass -y through
217 test/spec-bin.sh install-shells-with-apt "$@"
218}
219
220install-ubuntu-packages() {
221 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
222 install-debian-packages "$@"
223}
224
225wedge-deps-debian() {
226 # Install packages without prompt
227
228 # 2024-02 - there was an Ubuntu update, and we started needing this
229 sudo apt-get -y update
230
231 install-debian-packages -y
232}
233
234wedge-deps-fedora() {
235 # https://linuxconfig.org/install-development-tools-on-redhat-8
236 # Trying to get past compile errors
237 # sudo dnf group install --assumeyes 'Development Tools'
238
239 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
240}
241
242wedge-deps-alpine() {
243 # https://linuxconfig.org/install-development-tools-on-redhat-8
244 # Trying to get past compile errors
245 # sudo dnf group install --assumeyes 'Development Tools'
246
247 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
248}
249
250#
251# Unused patch, was experiment for Fedora
252#
253
254get-typed-ast-patch() {
255 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
256}
257
258# Work around typed_ast bug:
259# https://github.com/python/typed_ast/issues/169
260#
261# Apply this patch
262# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
263#
264# typed_ast is tarred up though
265patch-typed-ast() {
266 local package_dir=_cache/py3-libs
267 local patch=$PWD/deps/typed_ast.patch
268
269 pushd $package_dir
270 cat $patch
271 echo
272
273 local dir=typed_ast-1.4.3
274 local tar=typed_ast-1.4.3.tar.gz
275
276 echo OLD
277 ls -l $tar
278 echo
279
280 rm -r -f -v $dir
281 tar -x -z < $tar
282
283 pushd $dir
284 patch -p1 < $patch
285 popd
286 #find $dir
287
288 # Create a new one
289 tar --create --gzip --file $tar typed_ast-1.4.3
290
291 echo NEW
292 ls -l $tar
293 echo
294
295 popd
296}
297
298#
299# Fetch
300#
301
302download-to() {
303 local dir=$1
304 local url=$2
305 wget --no-clobber --directory-prefix "$dir" "$url"
306}
307
308maybe-extract() {
309 local wedge_dir=$1
310 local tar_name=$2
311 local out_dir=$3
312
313 if test -d "$wedge_dir/$out_dir"; then
314 log "Not extracting because $wedge_dir/$out_dir exists"
315 return
316 fi
317
318 local tar=$wedge_dir/$tar_name
319 case $tar_name in
320 *.gz|*.tgz) # mksh ends with .tgz
321 flag='--gzip'
322 ;;
323 *.bz2)
324 flag='--bzip2'
325 ;;
326 *.xz)
327 flag='--xz'
328 ;;
329 *)
330 die "tar with unknown extension: $tar_name"
331 ;;
332 esac
333
334 tar --extract $flag --file $tar --directory $wedge_dir
335}
336
337clone-mypy() {
338 ### replaces deps/from-git
339 local dest_dir=$1
340 local version=${2:-$MYPY_VERSION}
341
342 local dest=$dest_dir/mypy-$version
343 if test -d $dest; then
344 log "Not cloning because $dest exists"
345 return
346 fi
347
348 # v$VERSION is a tag, not a branch
349
350 # size optimization: --depth=1 --shallow-submodules
351 # https://git-scm.com/docs/git-clone
352
353 git clone --recursive --branch v$version \
354 --depth=1 --shallow-submodules \
355 $MYPY_GIT_URL $dest
356
357 # TODO: verify commit checksum
358}
359
360copy-source-medo() {
361 mkdir -p $DEPS_SOURCE_DIR
362
363 # Copy the whole tree, including the .treeptr files
364 cp --verbose --recursive --no-target-directory \
365 deps/source.medo/ $DEPS_SOURCE_DIR/
366}
367
368fetch() {
369 local py_only=${1:-}
370
371 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
372 # would do: fetch compressed tarballs designated by .treeptr files, and
373 # expand them.
374
375 # _build/deps-source/
376 # re2c/
377 # WEDGE
378 # re2c-3.0/ # expanded .tar.xz file
379
380 copy-source-medo
381
382 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
383 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
384 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
385 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
386
387 if test -n "$py_only"; then
388 log "Fetched dependencies for 'build/py.sh'"
389 return
390 fi
391
392 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
393 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
394 pyflakes-$PYFLAKES_VERSION
395
396 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
397 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
398 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
399 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
400
401 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
402 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" dash-$BASH_VER
403
404 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
405 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
406
407 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
408 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
409
410 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
411 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
412
413 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
414 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
415
416 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
417 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$DASH_VERSION
418
419 # Patch: this tarball doesn't follow the convention $name-$version
420 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
421 pushd $DEPS_SOURCE_DIR/mksh
422 mv -v mksh mksh-$MKSH_VERSION
423 popd
424 fi
425
426 # bloaty and uftrace are for benchmarks, in containers
427 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
428 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
429 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
430 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
431
432 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
433
434 # It's also copied into a wedge in install-wedges.
435 clone-mypy $DEPS_SOURCE_DIR/mypy
436
437 download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
438 maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
439
440 if command -v tree > /dev/null; then
441 tree -L 2 $DEPS_SOURCE_DIR
442 fi
443}
444
445fetch-py() {
446 fetch py_only
447}
448
449mirror-pyflakes() {
450 ### Workaround for network error during release
451 scp \
452 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
453 oilshell.org:oilshell.org/blob/
454}
455
456wedge-exists() {
457 ### Does an installed wedge already exist?
458
459 local name=$1
460 local version=$2
461 local wedge_dir=${3:-/wedge/oils-for-unix.org}
462
463 local installed=$wedge_dir/pkg/$name/$version
464
465 if test -d $installed; then
466 log "$installed already exists"
467 return 0
468 else
469 return 1
470 fi
471}
472
473#
474# Install
475#
476
477# TODO: py3-libs needs to be a WEDGE, so that that you can run
478# 'wedge build deps/source.medo/py3-libs/' and then get it in
479#
480# _build/wedge/{absolute,relative} # which one?
481#
482# It needs a BUILD DEPENDENCY on:
483# - the python3 wedge, so you can do python3 -m pip install.
484# - the mypy repo, which has test-requirements.txt
485
486download-py3-libs() {
487 ### Download source/binary packages, AFTER python3 is installed
488
489 # Note that this is NOT source code; there is binary code, e.g. in
490 # lxml-*.whl
491
492 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
493 local py_package_dir=_cache/py3-libs
494 mkdir -p $py_package_dir
495
496 # Avoids a warning, but doesn't fix typed_ast
497 #python3 -m pip download -d $py_package_dir wheel
498
499 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
500 python3 -m pip download -d $py_package_dir pexpect
501}
502
503install-py3-libs-in-venv() {
504 local venv_dir=$1
505 local mypy_dir=$2 # This is a param for host build vs. container build
506 local package_dir=_cache/py3-libs
507
508 source $venv_dir/bin/activate # enter virtualenv
509
510 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
511 # conflicts with MyPy deps!
512 # "ERROR: pip's dependency resolver does not currently take into account all
513 # the packages that are installed."
514
515 # --find-links uses a "cache dir" for packages (weird flag name!)
516
517 # Avoids a warning, but doesn't fix typed_ast
518 #time python3 -m pip install --find-links $package_dir wheel
519
520 # for mycpp/
521 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
522
523 # pexpect: for spec/stateful/*.py
524 time python3 -m pip install --find-links $package_dir pexpect
525}
526
527install-py3-libs-from-cache() {
528
529 # As well as end users
530
531 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
532
533 local py3
534 py3=$(command -v python3)
535 case $py3 in
536 *wedge/oils-for-unix.org/*)
537 ;;
538 *)
539 die "python3 is '$py3', but expected it to be in a wedge"
540 ;;
541 esac
542
543 log "Ensuring pip is installed (interpreter $(command -v python3)"
544 python3 -m ensurepip
545
546 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
547 log "Creating venv in $venv_dir"
548
549 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
550 # the /wedge we just built
551 python3 -m venv $venv_dir
552
553 log "Installing MyPy deps in venv"
554
555 # Run in a subshell because it mutates shell state
556 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
557}
558
559install-py3-libs() {
560 ### Invoked by Dockerfile.cpp-small, etc.
561
562 download-py3-libs
563 install-py3-libs-from-cache
564}
565
566# OBSOLETE in favor of install-spec-bin-fast
567install-spec-bin() {
568 if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
569 deps/wedge.sh unboxed-build _build/deps-source/dash
570 fi
571
572 if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
573 deps/wedge.sh unboxed-build _build/deps-source/mksh
574 fi
575
576 if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
577 deps/wedge.sh unboxed-build _build/deps-source/busybox
578 fi
579
580 # Fedora compile error - count_all_jobs
581 if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
582 deps/wedge.sh unboxed-build _build/deps-source/bash
583 fi
584
585 # Fedora compiler error
586 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
587 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
588 # like ncurses-devel should fix this, but it doesn't
589 #
590 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
591 #
592 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
593 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
594 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
595 #
596 # I think the ./configure is out of sync with the actual build?
597
598 if ! wedge-exists zsh $ZSH_VERSION ''; then
599 deps/wedge.sh unboxed-build _build/deps-source/zsh
600 fi
601
602 return
603
604 # Hm this has problem with out-of-tree build? I think Oils does too actually
605 if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
606 deps/wedge.sh unboxed-build _build/deps-source/yash
607 fi
608}
609
610# TODO:
611# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
612# already in each WEDGE file
613
614py-wedges() {
615 ### for build/py.sh all
616
617 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
618 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
619 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
620 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
621}
622
623cpp-wedges() {
624 ### for ninja / mycpp translation
625
626 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
627 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
628 echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
629
630 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
631 # separately for now
632 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
633}
634
635spec-bin-wedges() {
636 ### for test/spec-py.sh osh-all
637
638 echo dash $DASH_VERSION $USER_WEDGE_DIR
639 echo bash $BASH_VER $USER_WEDGE_DIR
640 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
641 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
642 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
643}
644
645timestamp() {
646 date '+%H:%M:%S'
647}
648
649my-time-tsv() {
650 python3 benchmarks/time_.py \
651 --tsv \
652 --time-span --rusage \
653 "$@"
654}
655
656maybe-install-wedge() {
657 local name=$1
658 local version=$2
659 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
660
661 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
662 local log_file=$WEDGE_LOG_DIR/$name.log.txt
663
664 echo " TASK $(timestamp) $name $version > $log_file"
665
666 # python3 because it's OUTSIDE the container
667 # Separate columns that could be joined: number of files, total size
668 my-time-tsv --print-header \
669 --field xargs_slot \
670 --field wedge \
671 --field wedge_HREF \
672 --field version \
673 --output $task_file
674
675 if wedge-exists "$name" "$version" "$wedge_dir"; then
676 echo "CACHED $(timestamp) $name $version"
677 return
678 fi
679
680 local -a cmd=( deps/wedge.sh unboxed-build _build/deps-source/$name/ )
681
682 set +o errexit
683 my-time-tsv \
684 --field "$XARGS_SLOT" \
685 --field "$name" \
686 --field "$name.log.txt" \
687 --field "$version" \
688 --append \
689 --output $task_file \
690 "${cmd[@]}" "$@" >$log_file 2>&1
691 local status=$?
692 set -o errexit
693
694 if test "$status" -eq 0; then
695 echo " OK $(timestamp) $name $version"
696 else
697 echo " FAIL $(timestamp) $name $version"
698 fi
699}
700
701dummy-task() {
702 ### For testing log capture
703 local name=$1
704 local version=$2
705
706 echo "Building $name $version"
707
708 # random float between 0 and 3
709 # weirdly we need a seed from bash
710 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
711 local secs
712 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
713
714 echo "sleep $secs"
715 sleep $secs
716
717 echo 'stdout'
718 log 'stderr'
719
720 if test $name = 'mksh'; then
721 echo "simulate failure for $name"
722 exit 2
723 fi
724}
725
726readonly WEDGE_LOG_DIR=_build/wedge/logs
727
728dummy-task-wrapper() {
729 # Similar to test/common.sh run-task-with-status, used by
730 # test/{spec,wild}-runner.sh
731 local name=$1
732 local version=$2
733
734 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
735 local log_file=$WEDGE_LOG_DIR/$name.log.txt
736
737 echo " TASK $(timestamp) $name $version > $log_file"
738
739 # python3 because it's OUTSIDE the container
740 # Separate columns that could be joined: number of files, total size
741 my-time-tsv --print-header \
742 --field xargs_slot \
743 --field wedge \
744 --field wedge_HREF \
745 --field version \
746 --output $task_file
747
748 my-time-tsv \
749 --field "$XARGS_SLOT" \
750 --field "$name" \
751 --field "$name.log.txt" \
752 --field "$version" \
753 --append \
754 --output $task_file \
755 $0 dummy-task "$@" >$log_file 2>&1 || true
756
757 echo " DONE $(timestamp) $name $version"
758}
759
760html-head() {
761 # python3 because we're outside containers
762 PYTHONPATH=. python3 doctools/html_head.py "$@"
763}
764
765index-html() {
766 local tasks_tsv=$1
767
768 local base_url='../../../web'
769 html-head --title 'Wedge Builds' \
770 "$base_url/ajax.js" \
771 "$base_url/table/table-sort.js" \
772 "$base_url/table/table-sort.css" \
773 "$base_url/base.css"\
774
775 table-sort-begin 'width60'
776
777 cat <<EOF
778 <p id="home-link">
779 <a href="/">oilshell.org</a>
780 </p>
781
782 <h1>Wedge Builds</h1>
783EOF
784
785 tsv2html3 $tasks_tsv
786
787 cat <<EOF
788 <p>
789 <a href="tasks.tsv">tasks.tsv</a>
790 </p>
791EOF
792
793 table-sort-end 'tasks' # ID for sorting
794}
795
796NPROC=$(nproc)
797#NPROC=1
798
799install-wedge-list() {
800 ### Reads task rows from stdin
801 local parallel=${1:-}
802
803
804 mkdir -p _build/wedge/logs
805
806 local -a flags
807 if test -n "$parallel"; then
808 log ""
809 log "=== Installing wedges with $NPROC jobs in parallel"
810 log ""
811 flags=( -P $NPROC )
812 else
813 log ""
814 log "=== Installing wedges serially"
815 log ""
816 fi
817
818 # Reads from stdin
819 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
820 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
821
822 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
823}
824
825write-task-report() {
826 local tasks_tsv=_build/wedge/logs/tasks.tsv
827
828 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
829 log "Wrote $tasks_tsv"
830
831 # TODO: version can be right-justified?
832 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
833column_name type precision strftime
834status integer 0 -
835elapsed_secs float 1 -
836user_secs float 1 -
837start_time float 1 %H:%M:%S
838end_time float 1 %H:%M:%S
839sys_secs float 1 -
840max_rss_KiB integer 0 -
841xargs_slot integer 0 -
842wedge string 0 -
843wedge_HREF string 0 -
844version string 0 -
845EOF
846
847 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
848 log "Wrote $WEDGE_LOG_DIR/index.html"
849}
850
851install-spec-bin-fast() {
852 spec-bin-wedges | install-wedge-list T
853 write-task-report
854}
855
856fake-py3-libs-wedge() {
857 local name=py3-libs
858 local version=$PY3_LIBS_VERSION
859
860 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
861 local log_file=$WEDGE_LOG_DIR/$name.log.txt
862
863 my-time-tsv --print-header \
864 --field xargs_slot \
865 --field wedge \
866 --field wedge_HREF \
867 --field version \
868 --output $task_file
869
870 # There is no xargs slot!
871 my-time-tsv \
872 --field "-1" \
873 --field "$name" \
874 --field "$name.log.txt" \
875 --field "$version" \
876 --append \
877 --output $task_file \
878 $0 install-py3-libs >$log_file 2>&1 || true
879
880 echo " FAKE $(timestamp) $name $version"
881}
882
883install-wedges-fast() {
884 echo " START $(timestamp)"
885
886 # Do all of them in parallel
887 { py-wedges; cpp-wedges; spec-bin-wedges; } | install-wedge-list T
888
889 fake-py3-libs-wedge
890 echo " END $(timestamp)"
891
892 write-task-report
893}
894
895# OBSOLETE in favor of install-wedges-fast
896install-wedges() {
897 local py_only=${1:-}
898
899 # TODO:
900 # - Make all of these RELATIVE wedges
901 # - Add
902 # - unboxed-rel-smoke-test -- move it inside container
903 # - rel-smoke-test -- mount it in a different location
904
905 if ! wedge-exists cmark $CMARK_VERSION; then
906 deps/wedge.sh unboxed-build _build/deps-source/cmark/
907 fi
908
909 if ! wedge-exists re2c $RE2C_VERSION; then
910 deps/wedge.sh unboxed-build _build/deps-source/re2c/
911 fi
912
913 if ! wedge-exists python2 $PY2_VERSION; then
914 deps/wedge.sh unboxed-build _build/deps-source/python2/
915 fi
916
917 if test -n "$py_only"; then
918 log "Installed dependencies for 'build/py.sh'"
919 return
920 fi
921
922 # Just copy this source tarball
923 if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
924 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
925 mkdir -p $dest_dir
926
927 cp --verbose --recursive --no-target-directory \
928 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
929 fi
930
931 if ! wedge-exists python3 $PY3_VERSION; then
932 deps/wedge.sh unboxed-build _build/deps-source/python3/
933 fi
934
935 # Copy all the contents, except for .git folder.
936 if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
937
938 # NOTE: We have to also copy the .git dir, because it has
939 # .git/modules/typeshed
940 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
941 mkdir -p $dest_dir
942
943 # Note: pack files in .git/modules/typeshed/objects/pack are read-only
944 # this can fail
945 cp --verbose --recursive --no-target-directory \
946 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
947 fi
948
949 if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
950 download-py3-libs
951 # This patch doesn't work?
952 # patch-typed-ast
953 install-py3-libs
954 fi
955
956 if ! wedge-exists souffle $SOUFFLE_VERSION $USER_WEDGE_DIR; then
957 deps/wedge.sh unboxed-build _build/deps-source/souffle/
958 fi
959
960 if command -v tree > /dev/null; then
961 tree -L 3 $USER_WEDGE_DIR
962 echo
963 tree -L 3 /wedge/oils-for-unix.org
964 fi
965}
966
967install-wedges-py() {
968 install-wedges py_only
969}
970
971#
972# Unboxed wedge builds
973#
974
975uftrace-host() {
976 ### built on demand; run $0 first
977
978 # BUG: doesn't detect python3
979 # WEDGE tells me that it depends on pkg-config
980 # 'apt-get install pkgconf' gets it
981 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
982
983 deps/wedge.sh unboxed-build _build/deps-source/uftrace
984}
985
986bloaty-host() {
987 deps/wedge.sh unboxed-build _build/deps-source/bloaty
988}
989
990R-libs-host() {
991 deps/wedge.sh unboxed-build _build/deps-source/R-libs
992}
993
994#
995# Wedges built inside a container, for copying into a container
996#
997
998container-wedges() {
999 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
1000
1001 #export-podman
1002
1003 if true; then
1004 deps/wedge.sh build deps/source.medo/time-helper
1005 deps/wedge.sh build deps/source.medo/cmark/
1006 deps/wedge.sh build deps/source.medo/re2c/
1007 deps/wedge.sh build deps/source.medo/python3/
1008 fi
1009
1010 if false; then
1011 deps/wedge.sh build deps/source.medo/bloaty/
1012 deps/wedge.sh build deps/source.medo/uftrace/
1013 fi
1014
1015 if false; then
1016 # For soil-benchmarks/ images
1017 deps/wedge.sh build deps/source.medo/R-libs/
1018 fi
1019}
1020
1021#
1022# Report
1023#
1024
1025commas() {
1026 # Wow I didn't know this :a trick
1027 #
1028 # OK this is a label and a loop, which makes sense. You can't do it with
1029 # pure regex.
1030 #
1031 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1032 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1033 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1034}
1035
1036wedge-sizes() {
1037 local tmp=_tmp/wedge-sizes.txt
1038
1039 # -b is --bytes, but use short flag for busybox compat
1040 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1041 { print $0 # print the line
1042 total_bytes += $1 # accumulate
1043 }
1044END { print total_bytes " TOTAL" }
1045' > $tmp
1046
1047 # printf justifies du output
1048 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1049 echo
1050
1051 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1052 #echo
1053}
1054
1055wedge-report() {
1056 # 4 levels deep shows the package
1057 if command -v tree > /dev/null; then
1058 tree -L 4 /wedge ~/wedge
1059 echo
1060 fi
1061
1062 wedge-sizes
1063
1064 local tmp=_tmp/wedge-manifest.txt
1065
1066 echo 'Biggest files'
1067 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1068 # busybox find doesn't have -printf
1069 echo 'find -printf failed'
1070 return
1071 fi
1072
1073 set +o errexit # ignore SIGPIPE
1074 sort -n --reverse $tmp | head -n 20 | commas
1075 set -o errexit
1076
1077 echo
1078
1079 # Show the most common file extensions
1080 #
1081 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1082 # most common
1083 #
1084 # There are also duplicate .a files for Python -- should look at how distros
1085 # get rid of those
1086
1087 cat $tmp | python3 -c '
1088import os, sys, collections
1089
1090bytes = collections.Counter()
1091files = collections.Counter()
1092
1093for line in sys.stdin:
1094 size, path = line.split(None, 1)
1095 path = path.strip() # remove newline
1096 _, ext = os.path.splitext(path)
1097 size = int(size)
1098
1099 bytes[ext] += size
1100 files[ext] += 1
1101
1102#print(bytes)
1103#print(files)
1104
1105n = 20
1106
1107print("Most common file types")
1108for ext, count in files.most_common()[:n]:
1109 print("%10d %s" % (count, ext))
1110
1111print()
1112
1113print("Total bytes by file type")
1114for ext, total_bytes in bytes.most_common()[:n]:
1115 print("%10d %s" % (total_bytes, ext))
1116' | commas
1117}
1118
1119run-task "$@"