OILS / build / deps.sh View on Github | oilshell.org

1115 lines, 588 significant
1#!/usr/bin/env bash
2#
3# Script for contributors to build dev dependencies -- packaged as cross-distro
4# "wedges". Tested in the Soil CI.
5#
6# Usage:
7# build/deps.sh <function name>
8#
9# Examples:
10# build/deps.sh fetch
11# build/deps.sh install-wedges # for both Python and C++
12# build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
13#
14# TODO: Do we need something faster, just python2, re2c, and cmark?
15#
16# - build/deps.sh fetch-py
17# - build/deps.sh install-wedges-py
18#
19# TODO: Can we make most of them non-root deps? This requires rebuilding
20# containers, which requires podman.
21#
22# rm -r -f ~/wedge # would be better
23
24set -o nounset
25set -o pipefail
26set -o errexit
27
28REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
29
30source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
31source deps/from-apt.sh # PY3_BUILD_DEPS
32#source deps/podman.sh
33source devtools/run-task.sh # run-task
34source test/tsv-lib.sh # tsv-concat
35source web/table/html.sh # table-sort-{begin,end}
36
37# Also in build/dev-shell.sh
38USER_WEDGE_DIR=~/wedge/oils-for-unix.org
39ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
40
41readonly DEPS_SOURCE_DIR=_build/deps-source
42
43readonly RE2C_VERSION=3.0
44readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
45
46readonly CMARK_VERSION=0.29.0
47readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
48
49readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
50
51readonly PY2_VERSION=2.7.18
52readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
53
54readonly PY3_VERSION=3.10.4
55readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
56
57readonly BASH_VER=4.4 # don't clobber BASH_VERSION
58readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
59
60readonly DASH_VERSION=0.5.10.2
61readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
62
63readonly ZSH_VERSION=5.1.1
64readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
65
66readonly MKSH_VERSION=R52c
67readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
68
69readonly BUSYBOX_VERSION='1.35.0'
70readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
71
72readonly YASH_VERSION=2.49
73readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
74
75readonly MYPY_GIT_URL=https://github.com/python/mypy
76readonly MYPY_VERSION=0.780
77
78readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
79
80# Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
81# https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
82readonly PYFLAKES_VERSION=2.4.0
83#readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
84# 2023-07: Mirrored to avoid network problem on broome during release
85readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
86
87readonly BLOATY_VERSION=1.1
88readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
89
90readonly UFTRACE_VERSION=0.13
91readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
92
93readonly SOUFFLE_VERSION=2.4.1
94readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
95
96log() {
97 echo "$0: $@" >& 2
98}
99
100die() {
101 log "$@"
102 exit 1
103}
104
105rm-oils-crap() {
106 ### When you want to start over
107
108 rm -r -f -v ~/wedge
109 sudo rm -r -f -v /wedge
110}
111
112# Note: git is an implicit dependency -- that's how we got the repo in the
113# first place!
114
115# python2-dev is no longer available on Debian 12
116# python-dev also seems gone
117#
118# wget: for fetching wedges (not on Debian by default!)
119# tree: tiny package that's useful for showing what we installed
120# g++: essential
121# libreadline-dev: needed for the build/prepare.sh Python build.
122# gawk: used by spec-runner.sh for the special match() function.
123# cmake: for cmark
124# PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
125# as well
126readonly -a WEDGE_DEPS_DEBIAN=(
127 bzip2
128 wget
129 tree
130 gawk
131 g++
132 ninja-build
133 cmake
134 libreadline-dev
135 systemtap-sdt-dev
136
137 # for Souffle, flex and bison
138 flex
139 bison
140
141 "${PY3_BUILD_DEPS[@]}"
142)
143
144readonly -a WEDGE_DEPS_ALPINE=(
145 bzip2
146 xz
147
148 wget tree gawk
149
150 gcc g++
151 ninja-build
152 # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
153 ninja-is-really-ninja
154 cmake
155
156 readline-dev
157 zlib-dev
158 libffi-dev
159 openssl-dev
160
161 ncurses-dev
162)
163
164readonly -a WEDGE_DEPS_FEDORA=(
165
166 # Weird, Fedora doesn't have these by default!
167 hostname
168 tar
169 bzip2
170
171 # https://packages.fedoraproject.org/pkgs/wget/wget/
172 wget
173 # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
174 tree
175 gawk
176
177 # https://packages.fedoraproject.org/pkgs/gcc/gcc/
178 gcc gcc-c++
179
180 ninja-build
181 cmake
182
183 readline-devel
184
185 # Like PY3_BUILD_DEPS
186 # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
187 zlib-devel
188 # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
189 libffi-devel
190 # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
191 openssl-devel
192
193 # For building zsh from source?
194 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
195 ncurses-devel
196 #libcap-devel
197
198 # still have a job control error compiling bash
199 # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
200 # glibc-devel
201)
202
203install-debian-packages() {
204 ### Packages for build/py.sh all, building wedges, etc.
205
206 set -x # show what needs sudo
207
208 # pass -y for say gitpod
209 sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
210 set +x
211
212 # maybe pass -y through
213 test/spec-bin.sh install-shells-with-apt "$@"
214}
215
216install-ubuntu-packages() {
217 ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
218 install-debian-packages "$@"
219}
220
221wedge-deps-debian() {
222 # Install packages without prompt
223
224 # 2024-02 - there was an Ubuntu update, and we started needing this
225 sudo apt-get -y update
226
227 install-debian-packages -y
228}
229
230wedge-deps-fedora() {
231 # https://linuxconfig.org/install-development-tools-on-redhat-8
232 # Trying to get past compile errors
233 # sudo dnf group install --assumeyes 'Development Tools'
234
235 sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
236}
237
238wedge-deps-alpine() {
239 # https://linuxconfig.org/install-development-tools-on-redhat-8
240 # Trying to get past compile errors
241 # sudo dnf group install --assumeyes 'Development Tools'
242
243 sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
244}
245
246#
247# Unused patch, was experiment for Fedora
248#
249
250get-typed-ast-patch() {
251 curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
252}
253
254# Work around typed_ast bug:
255# https://github.com/python/typed_ast/issues/169
256#
257# Apply this patch
258# https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
259#
260# typed_ast is tarred up though
261patch-typed-ast() {
262 local package_dir=_cache/py3-libs
263 local patch=$PWD/deps/typed_ast.patch
264
265 pushd $package_dir
266 cat $patch
267 echo
268
269 local dir=typed_ast-1.4.3
270 local tar=typed_ast-1.4.3.tar.gz
271
272 echo OLD
273 ls -l $tar
274 echo
275
276 rm -r -f -v $dir
277 tar -x -z < $tar
278
279 pushd $dir
280 patch -p1 < $patch
281 popd
282 #find $dir
283
284 # Create a new one
285 tar --create --gzip --file $tar typed_ast-1.4.3
286
287 echo NEW
288 ls -l $tar
289 echo
290
291 popd
292}
293
294#
295# Fetch
296#
297
298download-to() {
299 local dir=$1
300 local url=$2
301 wget --no-clobber --directory-prefix "$dir" "$url"
302}
303
304maybe-extract() {
305 local wedge_dir=$1
306 local tar_name=$2
307 local out_dir=$3
308
309 if test -d "$wedge_dir/$out_dir"; then
310 log "Not extracting because $wedge_dir/$out_dir exists"
311 return
312 fi
313
314 local tar=$wedge_dir/$tar_name
315 case $tar_name in
316 *.gz|*.tgz) # mksh ends with .tgz
317 flag='--gzip'
318 ;;
319 *.bz2)
320 flag='--bzip2'
321 ;;
322 *.xz)
323 flag='--xz'
324 ;;
325 *)
326 die "tar with unknown extension: $tar_name"
327 ;;
328 esac
329
330 tar --extract $flag --file $tar --directory $wedge_dir
331}
332
333clone-mypy() {
334 ### replaces deps/from-git
335 local dest_dir=$1
336 local version=${2:-$MYPY_VERSION}
337
338 local dest=$dest_dir/mypy-$version
339 if test -d $dest; then
340 log "Not cloning because $dest exists"
341 return
342 fi
343
344 # v$VERSION is a tag, not a branch
345
346 # size optimization: --depth=1 --shallow-submodules
347 # https://git-scm.com/docs/git-clone
348
349 git clone --recursive --branch v$version \
350 --depth=1 --shallow-submodules \
351 $MYPY_GIT_URL $dest
352
353 # TODO: verify commit checksum
354}
355
356copy-source-medo() {
357 mkdir -p $DEPS_SOURCE_DIR
358
359 # Copy the whole tree, including the .treeptr files
360 cp --verbose --recursive --no-target-directory \
361 deps/source.medo/ $DEPS_SOURCE_DIR/
362}
363
364fetch() {
365 local py_only=${1:-}
366
367 # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
368 # would do: fetch compressed tarballs designated by .treeptr files, and
369 # expand them.
370
371 # _build/deps-source/
372 # re2c/
373 # WEDGE
374 # re2c-3.0/ # expanded .tar.xz file
375
376 copy-source-medo
377
378 download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
379 download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
380 maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
381 maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
382
383 if test -n "$py_only"; then
384 log "Fetched dependencies for 'build/py.sh'"
385 return
386 fi
387
388 download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
389 maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
390 pyflakes-$PYFLAKES_VERSION
391
392 download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
393 download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
394 maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
395 maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
396
397 download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
398 maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" dash-$BASH_VER
399
400 download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
401 maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
402
403 download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
404 maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
405
406 download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
407 maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
408
409 download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
410 maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
411
412 download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
413 maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$DASH_VERSION
414
415 # Patch: this tarball doesn't follow the convention $name-$version
416 if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
417 pushd $DEPS_SOURCE_DIR/mksh
418 mv -v mksh mksh-$MKSH_VERSION
419 popd
420 fi
421
422 # bloaty and uftrace are for benchmarks, in containers
423 download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
424 download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
425 maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
426 maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
427
428 # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
429
430 # It's also copied into a wedge in install-wedges.
431 clone-mypy $DEPS_SOURCE_DIR/mypy
432
433 download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
434 maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
435
436 if command -v tree > /dev/null; then
437 tree -L 2 $DEPS_SOURCE_DIR
438 fi
439}
440
441fetch-py() {
442 fetch py_only
443}
444
445mirror-pyflakes() {
446 ### Workaround for network error during release
447 scp \
448 $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
449 oilshell.org:oilshell.org/blob/
450}
451
452wedge-exists() {
453 ### Does an installed wedge already exist?
454
455 local name=$1
456 local version=$2
457 local wedge_dir=${3:-/wedge/oils-for-unix.org}
458
459 local installed=$wedge_dir/pkg/$name/$version
460
461 if test -d $installed; then
462 log "$installed already exists"
463 return 0
464 else
465 return 1
466 fi
467}
468
469#
470# Install
471#
472
473# TODO: py3-libs needs to be a WEDGE, so that that you can run
474# 'wedge build deps/source.medo/py3-libs/' and then get it in
475#
476# _build/wedge/{absolute,relative} # which one?
477#
478# It needs a BUILD DEPENDENCY on:
479# - the python3 wedge, so you can do python3 -m pip install.
480# - the mypy repo, which has test-requirements.txt
481
482download-py3-libs() {
483 ### Download source/binary packages, AFTER python3 is installed
484
485 # Note that this is NOT source code; there is binary code, e.g. in
486 # lxml-*.whl
487
488 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
489 local py_package_dir=_cache/py3-libs
490 mkdir -p $py_package_dir
491
492 # Avoids a warning, but doesn't fix typed_ast
493 #python3 -m pip download -d $py_package_dir wheel
494
495 python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
496 python3 -m pip download -d $py_package_dir pexpect
497}
498
499install-py3-libs-in-venv() {
500 local venv_dir=$1
501 local mypy_dir=$2 # This is a param for host build vs. container build
502 local package_dir=_cache/py3-libs
503
504 source $venv_dir/bin/activate # enter virtualenv
505
506 # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
507 # conflicts with MyPy deps!
508 # "ERROR: pip's dependency resolver does not currently take into account all
509 # the packages that are installed."
510
511 # --find-links uses a "cache dir" for packages (weird flag name!)
512
513 # Avoids a warning, but doesn't fix typed_ast
514 #time python3 -m pip install --find-links $package_dir wheel
515
516 # for mycpp/
517 time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
518
519 # pexpect: for spec/stateful/*.py
520 time python3 -m pip install --find-links $package_dir pexpect
521}
522
523install-py3-libs-from-cache() {
524
525 # As well as end users
526
527 local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
528
529 local py3
530 py3=$(command -v python3)
531 case $py3 in
532 *wedge/oils-for-unix.org/*)
533 ;;
534 *)
535 die "python3 is '$py3', but expected it to be in a wedge"
536 ;;
537 esac
538
539 log "Ensuring pip is installed (interpreter $(command -v python3)"
540 python3 -m ensurepip
541
542 local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
543 log "Creating venv in $venv_dir"
544
545 # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
546 # the /wedge we just built
547 python3 -m venv $venv_dir
548
549 log "Installing MyPy deps in venv"
550
551 # Run in a subshell because it mutates shell state
552 $0 install-py3-libs-in-venv $venv_dir $mypy_dir
553}
554
555install-py3-libs() {
556 ### Invoked by Dockerfile.cpp-small, etc.
557
558 download-py3-libs
559 install-py3-libs-from-cache
560}
561
562# OBSOLETE in favor of install-spec-bin-fast
563install-spec-bin() {
564 if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
565 deps/wedge.sh unboxed-build _build/deps-source/dash
566 fi
567
568 if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
569 deps/wedge.sh unboxed-build _build/deps-source/mksh
570 fi
571
572 if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
573 deps/wedge.sh unboxed-build _build/deps-source/busybox
574 fi
575
576 # Fedora compile error - count_all_jobs
577 if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
578 deps/wedge.sh unboxed-build _build/deps-source/bash
579 fi
580
581 # Fedora compiler error
582 # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
583 # fallback in Src/Modules/termcap.c that causes a compile error! It seems
584 # like ncurses-devel should fix this, but it doesn't
585 #
586 # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
587 #
588 # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
589 # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
590 # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
591 #
592 # I think the ./configure is out of sync with the actual build?
593
594 if ! wedge-exists zsh $ZSH_VERSION ''; then
595 deps/wedge.sh unboxed-build _build/deps-source/zsh
596 fi
597
598 return
599
600 # Hm this has problem with out-of-tree build? I think Oils does too actually
601 if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
602 deps/wedge.sh unboxed-build _build/deps-source/yash
603 fi
604}
605
606# TODO:
607# - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
608# already in each WEDGE file
609
610py-wedges() {
611 ### for build/py.sh all
612
613 echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
614 echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
615 echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
616 echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
617}
618
619cpp-wedges() {
620 ### for ninja / mycpp translation
621
622 echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
623 echo mypy $MYPY_VERSION $USER_WEDGE_DIR
624 echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
625
626 # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
627 # separately for now
628 #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
629}
630
631spec-bin-wedges() {
632 ### for test/spec-py.sh osh-all
633
634 echo dash $DASH_VERSION $USER_WEDGE_DIR
635 echo bash $BASH_VER $USER_WEDGE_DIR
636 echo mksh $MKSH_VERSION $USER_WEDGE_DIR
637 echo zsh $ZSH_VERSION $USER_WEDGE_DIR
638 echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
639}
640
641timestamp() {
642 date '+%H:%M:%S'
643}
644
645my-time-tsv() {
646 python3 benchmarks/time_.py \
647 --tsv \
648 --time-span --rusage \
649 "$@"
650}
651
652maybe-install-wedge() {
653 local name=$1
654 local version=$2
655 local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
656
657 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
658 local log_file=$WEDGE_LOG_DIR/$name.log.txt
659
660 echo " TASK $(timestamp) $name $version > $log_file"
661
662 # python3 because it's OUTSIDE the container
663 # Separate columns that could be joined: number of files, total size
664 my-time-tsv --print-header \
665 --field xargs_slot \
666 --field wedge \
667 --field wedge_HREF \
668 --field version \
669 --output $task_file
670
671 if wedge-exists "$name" "$version" "$wedge_dir"; then
672 echo "CACHED $(timestamp) $name $version"
673 return
674 fi
675
676 local -a cmd=( deps/wedge.sh unboxed-build _build/deps-source/$name/ )
677
678 set +o errexit
679 my-time-tsv \
680 --field "$XARGS_SLOT" \
681 --field "$name" \
682 --field "$name.log.txt" \
683 --field "$version" \
684 --append \
685 --output $task_file \
686 "${cmd[@]}" "$@" >$log_file 2>&1
687 local status=$?
688 set -o errexit
689
690 if test "$status" -eq 0; then
691 echo " OK $(timestamp) $name $version"
692 else
693 echo " FAIL $(timestamp) $name $version"
694 fi
695}
696
697dummy-task() {
698 ### For testing log capture
699 local name=$1
700 local version=$2
701
702 echo "Building $name $version"
703
704 # random float between 0 and 3
705 # weirdly we need a seed from bash
706 # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
707 local secs
708 secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
709
710 echo "sleep $secs"
711 sleep $secs
712
713 echo 'stdout'
714 log 'stderr'
715
716 if test $name = 'mksh'; then
717 echo "simulate failure for $name"
718 exit 2
719 fi
720}
721
722readonly WEDGE_LOG_DIR=_build/wedge/logs
723
724dummy-task-wrapper() {
725 # Similar to test/common.sh run-task-with-status, used by
726 # test/{spec,wild}-runner.sh
727 local name=$1
728 local version=$2
729
730 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
731 local log_file=$WEDGE_LOG_DIR/$name.log.txt
732
733 echo " TASK $(timestamp) $name $version > $log_file"
734
735 # python3 because it's OUTSIDE the container
736 # Separate columns that could be joined: number of files, total size
737 my-time-tsv --print-header \
738 --field xargs_slot \
739 --field wedge \
740 --field wedge_HREF \
741 --field version \
742 --output $task_file
743
744 my-time-tsv \
745 --field "$XARGS_SLOT" \
746 --field "$name" \
747 --field "$name.log.txt" \
748 --field "$version" \
749 --append \
750 --output $task_file \
751 $0 dummy-task "$@" >$log_file 2>&1 || true
752
753 echo " DONE $(timestamp) $name $version"
754}
755
756html-head() {
757 # python3 because we're outside containers
758 PYTHONPATH=. python3 doctools/html_head.py "$@"
759}
760
761index-html() {
762 local tasks_tsv=$1
763
764 local base_url='../../../web'
765 html-head --title 'Wedge Builds' \
766 "$base_url/ajax.js" \
767 "$base_url/table/table-sort.js" \
768 "$base_url/table/table-sort.css" \
769 "$base_url/base.css"\
770
771 table-sort-begin 'width60'
772
773 cat <<EOF
774 <p id="home-link">
775 <a href="/">oilshell.org</a>
776 </p>
777
778 <h1>Wedge Builds</h1>
779EOF
780
781 tsv2html3 $tasks_tsv
782
783 cat <<EOF
784 <p>
785 <a href="tasks.tsv">tasks.tsv</a>
786 </p>
787EOF
788
789 table-sort-end 'tasks' # ID for sorting
790}
791
792NPROC=$(nproc)
793#NPROC=1
794
795install-wedge-list() {
796 ### Reads task rows from stdin
797 local parallel=${1:-}
798
799
800 mkdir -p _build/wedge/logs
801
802 local -a flags
803 if test -n "$parallel"; then
804 log ""
805 log "=== Installing wedges with $NPROC jobs in parallel"
806 log ""
807 flags=( -P $NPROC )
808 else
809 log ""
810 log "=== Installing wedges serially"
811 log ""
812 fi
813
814 # Reads from stdin
815 # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
816 xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
817
818 #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
819}
820
821write-task-report() {
822 local tasks_tsv=_build/wedge/logs/tasks.tsv
823
824 python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
825 log "Wrote $tasks_tsv"
826
827 # TODO: version can be right-justified?
828 here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
829column_name type precision strftime
830status integer 0 -
831elapsed_secs float 1 -
832user_secs float 1 -
833start_time float 1 %H:%M:%S
834end_time float 1 %H:%M:%S
835sys_secs float 1 -
836max_rss_KiB integer 0 -
837xargs_slot integer 0 -
838wedge string 0 -
839wedge_HREF string 0 -
840version string 0 -
841EOF
842
843 index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
844 log "Wrote $WEDGE_LOG_DIR/index.html"
845}
846
847install-spec-bin-fast() {
848 spec-bin-wedges | install-wedge-list T
849 write-task-report
850}
851
852fake-py3-libs-wedge() {
853 local name=py3-libs
854 local version=$PY3_LIBS_VERSION
855
856 local task_file=$WEDGE_LOG_DIR/$name.task.tsv
857 local log_file=$WEDGE_LOG_DIR/$name.log.txt
858
859 my-time-tsv --print-header \
860 --field xargs_slot \
861 --field wedge \
862 --field wedge_HREF \
863 --field version \
864 --output $task_file
865
866 # There is no xargs slot!
867 my-time-tsv \
868 --field "-1" \
869 --field "$name" \
870 --field "$name.log.txt" \
871 --field "$version" \
872 --append \
873 --output $task_file \
874 $0 install-py3-libs >$log_file 2>&1 || true
875
876 echo " FAKE $(timestamp) $name $version"
877}
878
879install-wedges-fast() {
880 echo " START $(timestamp)"
881
882 # Do all of them in parallel
883 { py-wedges; cpp-wedges; spec-bin-wedges; } | install-wedge-list T
884
885 fake-py3-libs-wedge
886 echo " END $(timestamp)"
887
888 write-task-report
889}
890
891# OBSOLETE in favor of install-wedges-fast
892install-wedges() {
893 local py_only=${1:-}
894
895 # TODO:
896 # - Make all of these RELATIVE wedges
897 # - Add
898 # - unboxed-rel-smoke-test -- move it inside container
899 # - rel-smoke-test -- mount it in a different location
900
901 if ! wedge-exists cmark $CMARK_VERSION; then
902 deps/wedge.sh unboxed-build _build/deps-source/cmark/
903 fi
904
905 if ! wedge-exists re2c $RE2C_VERSION; then
906 deps/wedge.sh unboxed-build _build/deps-source/re2c/
907 fi
908
909 if ! wedge-exists python2 $PY2_VERSION; then
910 deps/wedge.sh unboxed-build _build/deps-source/python2/
911 fi
912
913 if test -n "$py_only"; then
914 log "Installed dependencies for 'build/py.sh'"
915 return
916 fi
917
918 # Just copy this source tarball
919 if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
920 local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
921 mkdir -p $dest_dir
922
923 cp --verbose --recursive --no-target-directory \
924 $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
925 fi
926
927 if ! wedge-exists python3 $PY3_VERSION; then
928 deps/wedge.sh unboxed-build _build/deps-source/python3/
929 fi
930
931 # Copy all the contents, except for .git folder.
932 if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
933
934 # NOTE: We have to also copy the .git dir, because it has
935 # .git/modules/typeshed
936 local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
937 mkdir -p $dest_dir
938
939 # Note: pack files in .git/modules/typeshed/objects/pack are read-only
940 # this can fail
941 cp --verbose --recursive --no-target-directory \
942 $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
943 fi
944
945 if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
946 download-py3-libs
947 # This patch doesn't work?
948 # patch-typed-ast
949 install-py3-libs
950 fi
951
952 if ! wedge-exists souffle $SOUFFLE_VERSION $USER_WEDGE_DIR; then
953 deps/wedge.sh unboxed-build _build/deps-source/souffle/
954 fi
955
956 if command -v tree > /dev/null; then
957 tree -L 3 $USER_WEDGE_DIR
958 echo
959 tree -L 3 /wedge/oils-for-unix.org
960 fi
961}
962
963install-wedges-py() {
964 install-wedges py_only
965}
966
967#
968# Unboxed wedge builds
969#
970
971uftrace-host() {
972 ### built on demand; run $0 first
973
974 # BUG: doesn't detect python3
975 # WEDGE tells me that it depends on pkg-config
976 # 'apt-get install pkgconf' gets it
977 # TODO: Should use python3 WEDGE instead of SYSTEM python3?
978
979 deps/wedge.sh unboxed-build _build/deps-source/uftrace
980}
981
982bloaty-host() {
983 deps/wedge.sh unboxed-build _build/deps-source/bloaty
984}
985
986R-libs-host() {
987 deps/wedge.sh unboxed-build _build/deps-source/R-libs
988}
989
990#
991# Wedges built inside a container, for copying into a container
992#
993
994container-wedges() {
995 #### host _build/wedge/binary -> guest container /wedge or ~/wedge
996
997 #export-podman
998
999 if true; then
1000 deps/wedge.sh build deps/source.medo/time-helper
1001 deps/wedge.sh build deps/source.medo/cmark/
1002 deps/wedge.sh build deps/source.medo/re2c/
1003 deps/wedge.sh build deps/source.medo/python3/
1004 fi
1005
1006 if false; then
1007 deps/wedge.sh build deps/source.medo/bloaty/
1008 deps/wedge.sh build deps/source.medo/uftrace/
1009 fi
1010
1011 if false; then
1012 # For soil-benchmarks/ images
1013 deps/wedge.sh build deps/source.medo/R-libs/
1014 fi
1015}
1016
1017#
1018# Report
1019#
1020
1021commas() {
1022 # Wow I didn't know this :a trick
1023 #
1024 # OK this is a label and a loop, which makes sense. You can't do it with
1025 # pure regex.
1026 #
1027 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1028 # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
1029 sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
1030}
1031
1032wedge-sizes() {
1033 local tmp=_tmp/wedge-sizes.txt
1034
1035 # -b is --bytes, but use short flag for busybox compat
1036 du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
1037 { print $0 # print the line
1038 total_bytes += $1 # accumulate
1039 }
1040END { print total_bytes " TOTAL" }
1041' > $tmp
1042
1043 # printf justifies du output
1044 cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
1045 echo
1046
1047 #du -s --si /wedge/*/*/* ~/wedge/*/*/*
1048 #echo
1049}
1050
1051wedge-report() {
1052 # 4 levels deep shows the package
1053 if command -v tree > /dev/null; then
1054 tree -L 4 /wedge ~/wedge
1055 echo
1056 fi
1057
1058 wedge-sizes
1059
1060 local tmp=_tmp/wedge-manifest.txt
1061
1062 echo 'Biggest files'
1063 if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
1064 # busybox find doesn't have -printf
1065 echo 'find -printf failed'
1066 return
1067 fi
1068
1069 set +o errexit # ignore SIGPIPE
1070 sort -n --reverse $tmp | head -n 20 | commas
1071 set -o errexit
1072
1073 echo
1074
1075 # Show the most common file extensions
1076 #
1077 # I feel like we should be able to get rid of .a files? That's 92 MB, second
1078 # most common
1079 #
1080 # There are also duplicate .a files for Python -- should look at how distros
1081 # get rid of those
1082
1083 cat $tmp | python3 -c '
1084import os, sys, collections
1085
1086bytes = collections.Counter()
1087files = collections.Counter()
1088
1089for line in sys.stdin:
1090 size, path = line.split(None, 1)
1091 path = path.strip() # remove newline
1092 _, ext = os.path.splitext(path)
1093 size = int(size)
1094
1095 bytes[ext] += size
1096 files[ext] += 1
1097
1098#print(bytes)
1099#print(files)
1100
1101n = 20
1102
1103print("Most common file types")
1104for ext, count in files.most_common()[:n]:
1105 print("%10d %s" % (count, ext))
1106
1107print()
1108
1109print("Total bytes by file type")
1110for ext, total_bytes in bytes.most_common()[:n]:
1111 print("%10d %s" % (total_bytes, ext))
1112' | commas
1113}
1114
1115run-task "$@"