1 | #!/usr/bin/env bash
|
2 | #
|
3 | # Script for contributors to build dev dependencies -- packaged as cross-distro
|
4 | # "wedges". Tested in the Soil CI.
|
5 | #
|
6 | # Usage:
|
7 | # build/deps.sh <function name>
|
8 | #
|
9 | # Examples:
|
10 | # build/deps.sh fetch
|
11 | # build/deps.sh install-wedges # for both Python and C++
|
12 | # build/deps.sh rm-oils-crap # rm -r -f /wedge ~/wedge to start over
|
13 | #
|
14 | # TODO: Do we need something faster, just python2, re2c, and cmark?
|
15 | #
|
16 | # - build/deps.sh fetch-py
|
17 | # - build/deps.sh install-wedges-py
|
18 | #
|
19 | # TODO: Can we make most of them non-root deps? This requires rebuilding
|
20 | # containers, which requires podman.
|
21 | #
|
22 | # rm -r -f ~/wedge # would be better
|
23 |
|
24 | set -o nounset
|
25 | set -o pipefail
|
26 | set -o errexit
|
27 |
|
28 | REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
|
29 |
|
30 | source build/dev-shell.sh # python3 in PATH, PY3_LIBS_VERSION
|
31 | source deps/from-apt.sh # PY3_BUILD_DEPS
|
32 | #source deps/podman.sh
|
33 | source devtools/run-task.sh # run-task
|
34 | source test/tsv-lib.sh # tsv-concat
|
35 | source web/table/html.sh # table-sort-{begin,end}
|
36 |
|
37 | # Also in build/dev-shell.sh
|
38 | USER_WEDGE_DIR=~/wedge/oils-for-unix.org
|
39 | ROOT_WEDGE_DIR=/wedge/oils-for-unix.org
|
40 |
|
41 | readonly DEPS_SOURCE_DIR=_build/deps-source
|
42 |
|
43 | readonly RE2C_VERSION=3.0
|
44 | readonly RE2C_URL="https://github.com/skvadrik/re2c/releases/download/$RE2C_VERSION/re2c-$RE2C_VERSION.tar.xz"
|
45 |
|
46 | readonly CMARK_VERSION=0.29.0
|
47 | readonly CMARK_URL="https://github.com/commonmark/cmark/archive/$CMARK_VERSION.tar.gz"
|
48 |
|
49 | readonly PY_FTP_MIRROR="${PY_FTP_MIRROR:-https://www.python.org/ftp}"
|
50 |
|
51 | readonly PY2_VERSION=2.7.18
|
52 | readonly PY2_URL="$PY_FTP_MIRROR/python/$PY2_VERSION/Python-$PY2_VERSION.tar.xz"
|
53 |
|
54 | readonly PY3_VERSION=3.10.4
|
55 | readonly PY3_URL="$PY_FTP_MIRROR/python/$PY3_VERSION/Python-$PY3_VERSION.tar.xz"
|
56 |
|
57 | readonly BASH_VER=4.4 # don't clobber BASH_VERSION
|
58 | readonly BASH_URL="https://www.oilshell.org/blob/spec-bin/bash-$BASH_VER.tar.gz"
|
59 |
|
60 | readonly DASH_VERSION=0.5.10.2
|
61 | readonly DASH_URL="https://www.oilshell.org/blob/spec-bin/dash-$DASH_VERSION.tar.gz"
|
62 |
|
63 | readonly ZSH_VERSION=5.1.1
|
64 | readonly ZSH_URL="https://www.oilshell.org/blob/spec-bin/zsh-$ZSH_VERSION.tar.xz"
|
65 |
|
66 | readonly MKSH_VERSION=R52c
|
67 | readonly MKSH_URL="https://www.oilshell.org/blob/spec-bin/mksh-$MKSH_VERSION.tgz"
|
68 |
|
69 | readonly BUSYBOX_VERSION='1.35.0'
|
70 | readonly BUSYBOX_URL="https://www.oilshell.org/blob/spec-bin/busybox-$BUSYBOX_VERSION.tar.bz2"
|
71 |
|
72 | readonly YASH_VERSION=2.49
|
73 | readonly YASH_URL="https://www.oilshell.org/blob/spec-bin/yash-$YASH_VERSION.tar.xz"
|
74 |
|
75 | readonly MYPY_GIT_URL=https://github.com/python/mypy
|
76 | readonly MYPY_VERSION=0.780
|
77 |
|
78 | readonly PY3_LIBS=~/wedge/oils-for-unix.org/pkg/py3-libs/$MYPY_VERSION
|
79 |
|
80 | # Version 2.4.0 from 2021-10-06 was the last version that supported Python 2
|
81 | # https://github.com/PyCQA/pyflakes/blob/main/NEWS.rst
|
82 | readonly PYFLAKES_VERSION=2.4.0
|
83 | #readonly PYFLAKES_URL='https://files.pythonhosted.org/packages/15/60/c577e54518086e98470e9088278247f4af1d39cb43bcbd731e2c307acd6a/pyflakes-2.4.0.tar.gz'
|
84 | # 2023-07: Mirrored to avoid network problem on broome during release
|
85 | readonly PYFLAKES_URL='https://www.oilshell.org/blob/pyflakes-2.4.0.tar.gz'
|
86 |
|
87 | readonly BLOATY_VERSION=1.1
|
88 | readonly BLOATY_URL='https://github.com/google/bloaty/releases/download/v1.1/bloaty-1.1.tar.bz2'
|
89 |
|
90 | readonly UFTRACE_VERSION=0.13
|
91 | readonly UFTRACE_URL='https://github.com/namhyung/uftrace/archive/refs/tags/v0.13.tar.gz'
|
92 |
|
93 | readonly SOUFFLE_VERSION=2.4.1
|
94 | readonly SOUFFLE_URL=https://github.com/souffle-lang/souffle/archive/refs/tags/2.4.1.tar.gz
|
95 |
|
96 | log() {
|
97 | echo "$0: $@" >& 2
|
98 | }
|
99 |
|
100 | die() {
|
101 | log "$@"
|
102 | exit 1
|
103 | }
|
104 |
|
105 | rm-oils-crap() {
|
106 | ### When you want to start over
|
107 |
|
108 | rm -r -f -v ~/wedge
|
109 | sudo rm -r -f -v /wedge
|
110 | }
|
111 |
|
112 | # Note: git is an implicit dependency -- that's how we got the repo in the
|
113 | # first place!
|
114 |
|
115 | # python2-dev is no longer available on Debian 12
|
116 | # python-dev also seems gone
|
117 | #
|
118 | # wget: for fetching wedges (not on Debian by default!)
|
119 | # tree: tiny package that's useful for showing what we installed
|
120 | # g++: essential
|
121 | # libreadline-dev: needed for the build/prepare.sh Python build.
|
122 | # gawk: used by spec-runner.sh for the special match() function.
|
123 | # cmake: for cmark
|
124 | # PY3_BUILD_DEPS - I think these will be used for building the Python 2 wedge
|
125 | # as well
|
126 | readonly -a WEDGE_DEPS_DEBIAN=(
|
127 | bzip2
|
128 | wget
|
129 | tree
|
130 | gawk
|
131 | g++
|
132 | ninja-build
|
133 | cmake
|
134 | libreadline-dev
|
135 | systemtap-sdt-dev
|
136 |
|
137 | # for Souffle, flex and bison
|
138 | flex
|
139 | bison
|
140 |
|
141 | "${PY3_BUILD_DEPS[@]}"
|
142 | )
|
143 |
|
144 | readonly -a WEDGE_DEPS_ALPINE=(
|
145 | bzip2
|
146 | xz
|
147 |
|
148 | wget tree gawk
|
149 |
|
150 | gcc g++
|
151 | ninja-build
|
152 | # https://pkgs.alpinelinux.org/packages?name=ninja-is-really-ninja&branch=v3.19&repo=&arch=&maintainer=
|
153 | ninja-is-really-ninja
|
154 | cmake
|
155 |
|
156 | readline-dev
|
157 | zlib-dev
|
158 | libffi-dev
|
159 | openssl-dev
|
160 |
|
161 | ncurses-dev
|
162 | )
|
163 |
|
164 | readonly -a WEDGE_DEPS_FEDORA=(
|
165 |
|
166 | # Weird, Fedora doesn't have these by default!
|
167 | hostname
|
168 | tar
|
169 | bzip2
|
170 |
|
171 | # https://packages.fedoraproject.org/pkgs/wget/wget/
|
172 | wget
|
173 | # https://packages.fedoraproject.org/pkgs/tree-pkg/tree/
|
174 | tree
|
175 | gawk
|
176 |
|
177 | # https://packages.fedoraproject.org/pkgs/gcc/gcc/
|
178 | gcc gcc-c++
|
179 |
|
180 | ninja-build
|
181 | cmake
|
182 |
|
183 | readline-devel
|
184 |
|
185 | # Like PY3_BUILD_DEPS
|
186 | # https://packages.fedoraproject.org/pkgs/zlib/zlib-devel/
|
187 | zlib-devel
|
188 | # https://packages.fedoraproject.org/pkgs/libffi/libffi-devel/
|
189 | libffi-devel
|
190 | # https://packages.fedoraproject.org/pkgs/openssl/openssl-devel/
|
191 | openssl-devel
|
192 |
|
193 | # For building zsh from source?
|
194 | # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
|
195 | ncurses-devel
|
196 | #libcap-devel
|
197 |
|
198 | # still have a job control error compiling bash
|
199 | # https://packages.fedoraproject.org/pkgs/glibc/glibc-devel/
|
200 | # glibc-devel
|
201 | )
|
202 |
|
203 | install-debian-packages() {
|
204 | ### Packages for build/py.sh all, building wedges, etc.
|
205 |
|
206 | set -x # show what needs sudo
|
207 |
|
208 | # pass -y for say gitpod
|
209 | sudo apt "$@" install "${WEDGE_DEPS_DEBIAN[@]}"
|
210 | set +x
|
211 |
|
212 | # maybe pass -y through
|
213 | test/spec-bin.sh install-shells-with-apt "$@"
|
214 | }
|
215 |
|
216 | install-ubuntu-packages() {
|
217 | ### Debian and Ubuntu packages are the same; this function is suggested on the wiki
|
218 | install-debian-packages "$@"
|
219 | }
|
220 |
|
221 | wedge-deps-debian() {
|
222 | # Install packages without prompt
|
223 |
|
224 | # 2024-02 - there was an Ubuntu update, and we started needing this
|
225 | sudo apt-get -y update
|
226 |
|
227 | install-debian-packages -y
|
228 | }
|
229 |
|
230 | wedge-deps-fedora() {
|
231 | # https://linuxconfig.org/install-development-tools-on-redhat-8
|
232 | # Trying to get past compile errors
|
233 | # sudo dnf group install --assumeyes 'Development Tools'
|
234 |
|
235 | sudo dnf install --assumeyes "${WEDGE_DEPS_FEDORA[@]}"
|
236 | }
|
237 |
|
238 | wedge-deps-alpine() {
|
239 | # https://linuxconfig.org/install-development-tools-on-redhat-8
|
240 | # Trying to get past compile errors
|
241 | # sudo dnf group install --assumeyes 'Development Tools'
|
242 |
|
243 | sudo apk add "${WEDGE_DEPS_ALPINE[@]}"
|
244 | }
|
245 |
|
246 | #
|
247 | # Unused patch, was experiment for Fedora
|
248 | #
|
249 |
|
250 | get-typed-ast-patch() {
|
251 | curl -o deps/typed_ast.patch https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96.patch
|
252 | }
|
253 |
|
254 | # Work around typed_ast bug:
|
255 | # https://github.com/python/typed_ast/issues/169
|
256 | #
|
257 | # Apply this patch
|
258 | # https://github.com/python/typed_ast/commit/123286721923ae8f3885dbfbad94d6ca940d5c96
|
259 | #
|
260 | # typed_ast is tarred up though
|
261 | patch-typed-ast() {
|
262 | local package_dir=_cache/py3-libs
|
263 | local patch=$PWD/deps/typed_ast.patch
|
264 |
|
265 | pushd $package_dir
|
266 | cat $patch
|
267 | echo
|
268 |
|
269 | local dir=typed_ast-1.4.3
|
270 | local tar=typed_ast-1.4.3.tar.gz
|
271 |
|
272 | echo OLD
|
273 | ls -l $tar
|
274 | echo
|
275 |
|
276 | rm -r -f -v $dir
|
277 | tar -x -z < $tar
|
278 |
|
279 | pushd $dir
|
280 | patch -p1 < $patch
|
281 | popd
|
282 | #find $dir
|
283 |
|
284 | # Create a new one
|
285 | tar --create --gzip --file $tar typed_ast-1.4.3
|
286 |
|
287 | echo NEW
|
288 | ls -l $tar
|
289 | echo
|
290 |
|
291 | popd
|
292 | }
|
293 |
|
294 | #
|
295 | # Fetch
|
296 | #
|
297 |
|
298 | download-to() {
|
299 | local dir=$1
|
300 | local url=$2
|
301 | wget --no-clobber --directory-prefix "$dir" "$url"
|
302 | }
|
303 |
|
304 | maybe-extract() {
|
305 | local wedge_dir=$1
|
306 | local tar_name=$2
|
307 | local out_dir=$3
|
308 |
|
309 | if test -d "$wedge_dir/$out_dir"; then
|
310 | log "Not extracting because $wedge_dir/$out_dir exists"
|
311 | return
|
312 | fi
|
313 |
|
314 | local tar=$wedge_dir/$tar_name
|
315 | case $tar_name in
|
316 | *.gz|*.tgz) # mksh ends with .tgz
|
317 | flag='--gzip'
|
318 | ;;
|
319 | *.bz2)
|
320 | flag='--bzip2'
|
321 | ;;
|
322 | *.xz)
|
323 | flag='--xz'
|
324 | ;;
|
325 | *)
|
326 | die "tar with unknown extension: $tar_name"
|
327 | ;;
|
328 | esac
|
329 |
|
330 | tar --extract $flag --file $tar --directory $wedge_dir
|
331 | }
|
332 |
|
333 | clone-mypy() {
|
334 | ### replaces deps/from-git
|
335 | local dest_dir=$1
|
336 | local version=${2:-$MYPY_VERSION}
|
337 |
|
338 | local dest=$dest_dir/mypy-$version
|
339 | if test -d $dest; then
|
340 | log "Not cloning because $dest exists"
|
341 | return
|
342 | fi
|
343 |
|
344 | # v$VERSION is a tag, not a branch
|
345 |
|
346 | # size optimization: --depth=1 --shallow-submodules
|
347 | # https://git-scm.com/docs/git-clone
|
348 |
|
349 | git clone --recursive --branch v$version \
|
350 | --depth=1 --shallow-submodules \
|
351 | $MYPY_GIT_URL $dest
|
352 |
|
353 | # TODO: verify commit checksum
|
354 | }
|
355 |
|
356 | copy-source-medo() {
|
357 | mkdir -p $DEPS_SOURCE_DIR
|
358 |
|
359 | # Copy the whole tree, including the .treeptr files
|
360 | cp --verbose --recursive --no-target-directory \
|
361 | deps/source.medo/ $DEPS_SOURCE_DIR/
|
362 | }
|
363 |
|
364 | fetch() {
|
365 | local py_only=${1:-}
|
366 |
|
367 | # For now, simulate what 'medo expand deps/source.medo _build/deps-source'
|
368 | # would do: fetch compressed tarballs designated by .treeptr files, and
|
369 | # expand them.
|
370 |
|
371 | # _build/deps-source/
|
372 | # re2c/
|
373 | # WEDGE
|
374 | # re2c-3.0/ # expanded .tar.xz file
|
375 |
|
376 | copy-source-medo
|
377 |
|
378 | download-to $DEPS_SOURCE_DIR/re2c "$RE2C_URL"
|
379 | download-to $DEPS_SOURCE_DIR/cmark "$CMARK_URL"
|
380 | maybe-extract $DEPS_SOURCE_DIR/re2c "$(basename $RE2C_URL)" re2c-$RE2C_VERSION
|
381 | maybe-extract $DEPS_SOURCE_DIR/cmark "$(basename $CMARK_URL)" cmark-$CMARK_VERSION
|
382 |
|
383 | if test -n "$py_only"; then
|
384 | log "Fetched dependencies for 'build/py.sh'"
|
385 | return
|
386 | fi
|
387 |
|
388 | download-to $DEPS_SOURCE_DIR/pyflakes "$PYFLAKES_URL"
|
389 | maybe-extract $DEPS_SOURCE_DIR/pyflakes "$(basename $PYFLAKES_URL)" \
|
390 | pyflakes-$PYFLAKES_VERSION
|
391 |
|
392 | download-to $DEPS_SOURCE_DIR/python2 "$PY2_URL"
|
393 | download-to $DEPS_SOURCE_DIR/python3 "$PY3_URL"
|
394 | maybe-extract $DEPS_SOURCE_DIR/python2 "$(basename $PY2_URL)" Python-$PY2_VERSION
|
395 | maybe-extract $DEPS_SOURCE_DIR/python3 "$(basename $PY3_URL)" Python-$PY3_VERSION
|
396 |
|
397 | download-to $DEPS_SOURCE_DIR/bash "$BASH_URL"
|
398 | maybe-extract $DEPS_SOURCE_DIR/bash "$(basename $BASH_URL)" dash-$BASH_VER
|
399 |
|
400 | download-to $DEPS_SOURCE_DIR/dash "$DASH_URL"
|
401 | maybe-extract $DEPS_SOURCE_DIR/dash "$(basename $DASH_URL)" dash-$DASH_VERSION
|
402 |
|
403 | download-to $DEPS_SOURCE_DIR/zsh "$ZSH_URL"
|
404 | maybe-extract $DEPS_SOURCE_DIR/zsh "$(basename $ZSH_URL)" zsh-$ZSH_VERSION
|
405 |
|
406 | download-to $DEPS_SOURCE_DIR/mksh "$MKSH_URL"
|
407 | maybe-extract $DEPS_SOURCE_DIR/mksh "$(basename $MKSH_URL)" mksh-$MKSH_VERSION
|
408 |
|
409 | download-to $DEPS_SOURCE_DIR/busybox "$BUSYBOX_URL"
|
410 | maybe-extract $DEPS_SOURCE_DIR/busybox "$(basename $BUSYBOX_URL)" busybox-$BUSYBOX_VERSION
|
411 |
|
412 | download-to $DEPS_SOURCE_DIR/yash "$YASH_URL"
|
413 | maybe-extract $DEPS_SOURCE_DIR/yash "$(basename $YASH_URL)" yash-$DASH_VERSION
|
414 |
|
415 | # Patch: this tarball doesn't follow the convention $name-$version
|
416 | if test -d $DEPS_SOURCE_DIR/mksh/mksh; then
|
417 | pushd $DEPS_SOURCE_DIR/mksh
|
418 | mv -v mksh mksh-$MKSH_VERSION
|
419 | popd
|
420 | fi
|
421 |
|
422 | # bloaty and uftrace are for benchmarks, in containers
|
423 | download-to $DEPS_SOURCE_DIR/bloaty "$BLOATY_URL"
|
424 | download-to $DEPS_SOURCE_DIR/uftrace "$UFTRACE_URL"
|
425 | maybe-extract $DEPS_SOURCE_DIR/bloaty "$(basename $BLOATY_URL)" uftrace-$BLOATY_VERSION
|
426 | maybe-extract $DEPS_SOURCE_DIR/uftrace "$(basename $UFTRACE_URL)" bloaty-$UFTRACE_VERSION
|
427 |
|
428 | # This is in $DEPS_SOURCE_DIR to COPY into containers, which mycpp will directly import.
|
429 |
|
430 | # It's also copied into a wedge in install-wedges.
|
431 | clone-mypy $DEPS_SOURCE_DIR/mypy
|
432 |
|
433 | download-to $DEPS_SOURCE_DIR/souffle "$SOUFFLE_URL"
|
434 | maybe-extract $DEPS_SOURCE_DIR/souffle "$(basename $SOUFFLE_URL)" souffle-$SOUFFLE_VERSION
|
435 |
|
436 | if command -v tree > /dev/null; then
|
437 | tree -L 2 $DEPS_SOURCE_DIR
|
438 | fi
|
439 | }
|
440 |
|
441 | fetch-py() {
|
442 | fetch py_only
|
443 | }
|
444 |
|
445 | mirror-pyflakes() {
|
446 | ### Workaround for network error during release
|
447 | scp \
|
448 | $DEPS_SOURCE_DIR/pyflakes/"$(basename $PYFLAKES_URL)" \
|
449 | oilshell.org:oilshell.org/blob/
|
450 | }
|
451 |
|
452 | wedge-exists() {
|
453 | ### Does an installed wedge already exist?
|
454 |
|
455 | local name=$1
|
456 | local version=$2
|
457 | local wedge_dir=${3:-/wedge/oils-for-unix.org}
|
458 |
|
459 | local installed=$wedge_dir/pkg/$name/$version
|
460 |
|
461 | if test -d $installed; then
|
462 | log "$installed already exists"
|
463 | return 0
|
464 | else
|
465 | return 1
|
466 | fi
|
467 | }
|
468 |
|
469 | #
|
470 | # Install
|
471 | #
|
472 |
|
473 | # TODO: py3-libs needs to be a WEDGE, so that that you can run
|
474 | # 'wedge build deps/source.medo/py3-libs/' and then get it in
|
475 | #
|
476 | # _build/wedge/{absolute,relative} # which one?
|
477 | #
|
478 | # It needs a BUILD DEPENDENCY on:
|
479 | # - the python3 wedge, so you can do python3 -m pip install.
|
480 | # - the mypy repo, which has test-requirements.txt
|
481 |
|
482 | download-py3-libs() {
|
483 | ### Download source/binary packages, AFTER python3 is installed
|
484 |
|
485 | # Note that this is NOT source code; there is binary code, e.g. in
|
486 | # lxml-*.whl
|
487 |
|
488 | local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
|
489 | local py_package_dir=_cache/py3-libs
|
490 | mkdir -p $py_package_dir
|
491 |
|
492 | # Avoids a warning, but doesn't fix typed_ast
|
493 | #python3 -m pip download -d $py_package_dir wheel
|
494 |
|
495 | python3 -m pip download -d $py_package_dir -r $mypy_dir/test-requirements.txt
|
496 | python3 -m pip download -d $py_package_dir pexpect
|
497 | }
|
498 |
|
499 | install-py3-libs-in-venv() {
|
500 | local venv_dir=$1
|
501 | local mypy_dir=$2 # This is a param for host build vs. container build
|
502 | local package_dir=_cache/py3-libs
|
503 |
|
504 | source $venv_dir/bin/activate # enter virtualenv
|
505 |
|
506 | # 2023-07 note: we're installing yapf in a DIFFERENT venv, because it
|
507 | # conflicts with MyPy deps!
|
508 | # "ERROR: pip's dependency resolver does not currently take into account all
|
509 | # the packages that are installed."
|
510 |
|
511 | # --find-links uses a "cache dir" for packages (weird flag name!)
|
512 |
|
513 | # Avoids a warning, but doesn't fix typed_ast
|
514 | #time python3 -m pip install --find-links $package_dir wheel
|
515 |
|
516 | # for mycpp/
|
517 | time python3 -m pip install --find-links $package_dir -r $mypy_dir/test-requirements.txt
|
518 |
|
519 | # pexpect: for spec/stateful/*.py
|
520 | time python3 -m pip install --find-links $package_dir pexpect
|
521 | }
|
522 |
|
523 | install-py3-libs-from-cache() {
|
524 |
|
525 | # As well as end users
|
526 |
|
527 | local mypy_dir=${1:-$DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION}
|
528 |
|
529 | local py3
|
530 | py3=$(command -v python3)
|
531 | case $py3 in
|
532 | *wedge/oils-for-unix.org/*)
|
533 | ;;
|
534 | *)
|
535 | die "python3 is '$py3', but expected it to be in a wedge"
|
536 | ;;
|
537 | esac
|
538 |
|
539 | log "Ensuring pip is installed (interpreter $(command -v python3)"
|
540 | python3 -m ensurepip
|
541 |
|
542 | local venv_dir=$USER_WEDGE_DIR/pkg/py3-libs/$PY3_LIBS_VERSION
|
543 | log "Creating venv in $venv_dir"
|
544 |
|
545 | # Note: the bin/python3 in this venv is a symlink to python3 in $PATH, i.e.
|
546 | # the /wedge we just built
|
547 | python3 -m venv $venv_dir
|
548 |
|
549 | log "Installing MyPy deps in venv"
|
550 |
|
551 | # Run in a subshell because it mutates shell state
|
552 | $0 install-py3-libs-in-venv $venv_dir $mypy_dir
|
553 | }
|
554 |
|
555 | install-py3-libs() {
|
556 | ### Invoked by Dockerfile.cpp-small, etc.
|
557 |
|
558 | download-py3-libs
|
559 | install-py3-libs-from-cache
|
560 | }
|
561 |
|
562 | # OBSOLETE in favor of install-spec-bin-fast
|
563 | install-spec-bin() {
|
564 | if ! wedge-exists dash $DASH_VERSION $USER_WEDGE_DIR; then
|
565 | deps/wedge.sh unboxed-build _build/deps-source/dash
|
566 | fi
|
567 |
|
568 | if ! wedge-exists mksh $MKSH_VERSION $USER_WEDGE_DIR; then
|
569 | deps/wedge.sh unboxed-build _build/deps-source/mksh
|
570 | fi
|
571 |
|
572 | if ! wedge-exists busybox $BUSYBOX_VERSION $USER_WEDGE_DIR; then
|
573 | deps/wedge.sh unboxed-build _build/deps-source/busybox
|
574 | fi
|
575 |
|
576 | # Fedora compile error - count_all_jobs
|
577 | if ! wedge-exists bash $BASH_VER $USER_WEDGE_DIR; then
|
578 | deps/wedge.sh unboxed-build _build/deps-source/bash
|
579 | fi
|
580 |
|
581 | # Fedora compiler error
|
582 | # zsh ./configure is NOT detecting 'boolcodes', and then it has a broken
|
583 | # fallback in Src/Modules/termcap.c that causes a compile error! It seems
|
584 | # like ncurses-devel should fix this, but it doesn't
|
585 | #
|
586 | # https://koji.fedoraproject.org/koji/rpminfo?rpmID=36987813
|
587 | #
|
588 | # from /home/build/oil/_build/deps-source/zsh/zsh-5.1.1/Src/Modules/termcap.c:38:
|
589 | # /usr/include/term.h:783:56: note: previous declaration of ‘boolcodes’ with type ‘const char * const[]’
|
590 | # 783 | extern NCURSES_EXPORT_VAR(NCURSES_CONST char * const ) boolcodes[];
|
591 | #
|
592 | # I think the ./configure is out of sync with the actual build?
|
593 |
|
594 | if ! wedge-exists zsh $ZSH_VERSION ''; then
|
595 | deps/wedge.sh unboxed-build _build/deps-source/zsh
|
596 | fi
|
597 |
|
598 | return
|
599 |
|
600 | # Hm this has problem with out-of-tree build? I think Oils does too actually
|
601 | if ! wedge-exists yash $YASH_VERSION $USER_WEDGE_DIR; then
|
602 | deps/wedge.sh unboxed-build _build/deps-source/yash
|
603 | fi
|
604 | }
|
605 |
|
606 | # TODO:
|
607 | # - $ROOT_WEDGE_DIR vs. $USER_WEDGE_DIR is duplicating information that's
|
608 | # already in each WEDGE file
|
609 |
|
610 | py-wedges() {
|
611 | ### for build/py.sh all
|
612 |
|
613 | echo cmark $CMARK_VERSION $ROOT_WEDGE_DIR
|
614 | echo re2c $RE2C_VERSION $ROOT_WEDGE_DIR
|
615 | echo python2 $PY2_VERSION $ROOT_WEDGE_DIR
|
616 | echo pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR
|
617 | }
|
618 |
|
619 | cpp-wedges() {
|
620 | ### for ninja / mycpp translation
|
621 |
|
622 | echo python3 $PY3_VERSION $ROOT_WEDGE_DIR
|
623 | echo mypy $MYPY_VERSION $USER_WEDGE_DIR
|
624 | echo souffle $SOUFFLE_VERSION $USER_WEDGE_DIR
|
625 |
|
626 | # py3-libs has a built time dep on both python3 and MyPy, so we're doing it
|
627 | # separately for now
|
628 | #echo py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR
|
629 | }
|
630 |
|
631 | spec-bin-wedges() {
|
632 | ### for test/spec-py.sh osh-all
|
633 |
|
634 | echo dash $DASH_VERSION $USER_WEDGE_DIR
|
635 | echo bash $BASH_VER $USER_WEDGE_DIR
|
636 | echo mksh $MKSH_VERSION $USER_WEDGE_DIR
|
637 | echo zsh $ZSH_VERSION $USER_WEDGE_DIR
|
638 | echo busybox $BUSYBOX_VERSION $USER_WEDGE_DIR
|
639 | }
|
640 |
|
641 | timestamp() {
|
642 | date '+%H:%M:%S'
|
643 | }
|
644 |
|
645 | my-time-tsv() {
|
646 | python3 benchmarks/time_.py \
|
647 | --tsv \
|
648 | --time-span --rusage \
|
649 | "$@"
|
650 | }
|
651 |
|
652 | maybe-install-wedge() {
|
653 | local name=$1
|
654 | local version=$2
|
655 | local wedge_dir=$3 # e.g. $USER_WEDGE_DIR or empty
|
656 |
|
657 | local task_file=$WEDGE_LOG_DIR/$name.task.tsv
|
658 | local log_file=$WEDGE_LOG_DIR/$name.log.txt
|
659 |
|
660 | echo " TASK $(timestamp) $name $version > $log_file"
|
661 |
|
662 | # python3 because it's OUTSIDE the container
|
663 | # Separate columns that could be joined: number of files, total size
|
664 | my-time-tsv --print-header \
|
665 | --field xargs_slot \
|
666 | --field wedge \
|
667 | --field wedge_HREF \
|
668 | --field version \
|
669 | --output $task_file
|
670 |
|
671 | if wedge-exists "$name" "$version" "$wedge_dir"; then
|
672 | echo "CACHED $(timestamp) $name $version"
|
673 | return
|
674 | fi
|
675 |
|
676 | local -a cmd=( deps/wedge.sh unboxed-build _build/deps-source/$name/ )
|
677 |
|
678 | set +o errexit
|
679 | my-time-tsv \
|
680 | --field "$XARGS_SLOT" \
|
681 | --field "$name" \
|
682 | --field "$name.log.txt" \
|
683 | --field "$version" \
|
684 | --append \
|
685 | --output $task_file \
|
686 | "${cmd[@]}" "$@" >$log_file 2>&1
|
687 | local status=$?
|
688 | set -o errexit
|
689 |
|
690 | if test "$status" -eq 0; then
|
691 | echo " OK $(timestamp) $name $version"
|
692 | else
|
693 | echo " FAIL $(timestamp) $name $version"
|
694 | fi
|
695 | }
|
696 |
|
697 | dummy-task() {
|
698 | ### For testing log capture
|
699 | local name=$1
|
700 | local version=$2
|
701 |
|
702 | echo "Building $name $version"
|
703 |
|
704 | # random float between 0 and 3
|
705 | # weirdly we need a seed from bash
|
706 | # https://stackoverflow.com/questions/4048378/random-numbers-generation-with-awk-in-bash-shell
|
707 | local secs
|
708 | secs=$(awk -v seed=$RANDOM 'END { srand(seed); print rand() * 3 }' < /dev/null)
|
709 |
|
710 | echo "sleep $secs"
|
711 | sleep $secs
|
712 |
|
713 | echo 'stdout'
|
714 | log 'stderr'
|
715 |
|
716 | if test $name = 'mksh'; then
|
717 | echo "simulate failure for $name"
|
718 | exit 2
|
719 | fi
|
720 | }
|
721 |
|
722 | readonly WEDGE_LOG_DIR=_build/wedge/logs
|
723 |
|
724 | dummy-task-wrapper() {
|
725 | # Similar to test/common.sh run-task-with-status, used by
|
726 | # test/{spec,wild}-runner.sh
|
727 | local name=$1
|
728 | local version=$2
|
729 |
|
730 | local task_file=$WEDGE_LOG_DIR/$name.task.tsv
|
731 | local log_file=$WEDGE_LOG_DIR/$name.log.txt
|
732 |
|
733 | echo " TASK $(timestamp) $name $version > $log_file"
|
734 |
|
735 | # python3 because it's OUTSIDE the container
|
736 | # Separate columns that could be joined: number of files, total size
|
737 | my-time-tsv --print-header \
|
738 | --field xargs_slot \
|
739 | --field wedge \
|
740 | --field wedge_HREF \
|
741 | --field version \
|
742 | --output $task_file
|
743 |
|
744 | my-time-tsv \
|
745 | --field "$XARGS_SLOT" \
|
746 | --field "$name" \
|
747 | --field "$name.log.txt" \
|
748 | --field "$version" \
|
749 | --append \
|
750 | --output $task_file \
|
751 | $0 dummy-task "$@" >$log_file 2>&1 || true
|
752 |
|
753 | echo " DONE $(timestamp) $name $version"
|
754 | }
|
755 |
|
756 | html-head() {
|
757 | # python3 because we're outside containers
|
758 | PYTHONPATH=. python3 doctools/html_head.py "$@"
|
759 | }
|
760 |
|
761 | index-html() {
|
762 | local tasks_tsv=$1
|
763 |
|
764 | local base_url='../../../web'
|
765 | html-head --title 'Wedge Builds' \
|
766 | "$base_url/ajax.js" \
|
767 | "$base_url/table/table-sort.js" \
|
768 | "$base_url/table/table-sort.css" \
|
769 | "$base_url/base.css"\
|
770 |
|
771 | table-sort-begin 'width60'
|
772 |
|
773 | cat <<EOF
|
774 | <p id="home-link">
|
775 | <a href="/">oilshell.org</a>
|
776 | </p>
|
777 |
|
778 | <h1>Wedge Builds</h1>
|
779 | EOF
|
780 |
|
781 | tsv2html3 $tasks_tsv
|
782 |
|
783 | cat <<EOF
|
784 | <p>
|
785 | <a href="tasks.tsv">tasks.tsv</a>
|
786 | </p>
|
787 | EOF
|
788 |
|
789 | table-sort-end 'tasks' # ID for sorting
|
790 | }
|
791 |
|
792 | NPROC=$(nproc)
|
793 | #NPROC=1
|
794 |
|
795 | install-wedge-list() {
|
796 | ### Reads task rows from stdin
|
797 | local parallel=${1:-}
|
798 |
|
799 |
|
800 | mkdir -p _build/wedge/logs
|
801 |
|
802 | local -a flags
|
803 | if test -n "$parallel"; then
|
804 | log ""
|
805 | log "=== Installing wedges with $NPROC jobs in parallel"
|
806 | log ""
|
807 | flags=( -P $NPROC )
|
808 | else
|
809 | log ""
|
810 | log "=== Installing wedges serially"
|
811 | log ""
|
812 | fi
|
813 |
|
814 | # Reads from stdin
|
815 | # Note: --process-slot-var requires GNU xargs! busybox args doesn't have it.
|
816 | xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 maybe-install-wedge
|
817 |
|
818 | #xargs "${flags[@]}" -n 3 --process-slot-var=XARGS_SLOT -- $0 dummy-task-wrapper
|
819 | }
|
820 |
|
821 | write-task-report() {
|
822 | local tasks_tsv=_build/wedge/logs/tasks.tsv
|
823 |
|
824 | python3 devtools/tsv_concat.py $WEDGE_LOG_DIR/*.task.tsv > $tasks_tsv
|
825 | log "Wrote $tasks_tsv"
|
826 |
|
827 | # TODO: version can be right-justified?
|
828 | here-schema-tsv-4col >_build/wedge/logs/tasks.schema.tsv <<EOF
|
829 | column_name type precision strftime
|
830 | status integer 0 -
|
831 | elapsed_secs float 1 -
|
832 | user_secs float 1 -
|
833 | start_time float 1 %H:%M:%S
|
834 | end_time float 1 %H:%M:%S
|
835 | sys_secs float 1 -
|
836 | max_rss_KiB integer 0 -
|
837 | xargs_slot integer 0 -
|
838 | wedge string 0 -
|
839 | wedge_HREF string 0 -
|
840 | version string 0 -
|
841 | EOF
|
842 |
|
843 | index-html $tasks_tsv > $WEDGE_LOG_DIR/index.html
|
844 | log "Wrote $WEDGE_LOG_DIR/index.html"
|
845 | }
|
846 |
|
847 | install-spec-bin-fast() {
|
848 | spec-bin-wedges | install-wedge-list T
|
849 | write-task-report
|
850 | }
|
851 |
|
852 | fake-py3-libs-wedge() {
|
853 | local name=py3-libs
|
854 | local version=$PY3_LIBS_VERSION
|
855 |
|
856 | local task_file=$WEDGE_LOG_DIR/$name.task.tsv
|
857 | local log_file=$WEDGE_LOG_DIR/$name.log.txt
|
858 |
|
859 | my-time-tsv --print-header \
|
860 | --field xargs_slot \
|
861 | --field wedge \
|
862 | --field wedge_HREF \
|
863 | --field version \
|
864 | --output $task_file
|
865 |
|
866 | # There is no xargs slot!
|
867 | my-time-tsv \
|
868 | --field "-1" \
|
869 | --field "$name" \
|
870 | --field "$name.log.txt" \
|
871 | --field "$version" \
|
872 | --append \
|
873 | --output $task_file \
|
874 | $0 install-py3-libs >$log_file 2>&1 || true
|
875 |
|
876 | echo " FAKE $(timestamp) $name $version"
|
877 | }
|
878 |
|
879 | install-wedges-fast() {
|
880 | echo " START $(timestamp)"
|
881 |
|
882 | # Do all of them in parallel
|
883 | { py-wedges; cpp-wedges; spec-bin-wedges; } | install-wedge-list T
|
884 |
|
885 | fake-py3-libs-wedge
|
886 | echo " END $(timestamp)"
|
887 |
|
888 | write-task-report
|
889 | }
|
890 |
|
891 | # OBSOLETE in favor of install-wedges-fast
|
892 | install-wedges() {
|
893 | local py_only=${1:-}
|
894 |
|
895 | # TODO:
|
896 | # - Make all of these RELATIVE wedges
|
897 | # - Add
|
898 | # - unboxed-rel-smoke-test -- move it inside container
|
899 | # - rel-smoke-test -- mount it in a different location
|
900 |
|
901 | if ! wedge-exists cmark $CMARK_VERSION; then
|
902 | deps/wedge.sh unboxed-build _build/deps-source/cmark/
|
903 | fi
|
904 |
|
905 | if ! wedge-exists re2c $RE2C_VERSION; then
|
906 | deps/wedge.sh unboxed-build _build/deps-source/re2c/
|
907 | fi
|
908 |
|
909 | if ! wedge-exists python2 $PY2_VERSION; then
|
910 | deps/wedge.sh unboxed-build _build/deps-source/python2/
|
911 | fi
|
912 |
|
913 | if test -n "$py_only"; then
|
914 | log "Installed dependencies for 'build/py.sh'"
|
915 | return
|
916 | fi
|
917 |
|
918 | # Just copy this source tarball
|
919 | if ! wedge-exists pyflakes $PYFLAKES_VERSION $USER_WEDGE_DIR; then
|
920 | local dest_dir=$USER_WEDGE_DIR/pkg/pyflakes/$PYFLAKES_VERSION
|
921 | mkdir -p $dest_dir
|
922 |
|
923 | cp --verbose --recursive --no-target-directory \
|
924 | $DEPS_SOURCE_DIR/pyflakes/pyflakes-$PYFLAKES_VERSION $dest_dir
|
925 | fi
|
926 |
|
927 | if ! wedge-exists python3 $PY3_VERSION; then
|
928 | deps/wedge.sh unboxed-build _build/deps-source/python3/
|
929 | fi
|
930 |
|
931 | # Copy all the contents, except for .git folder.
|
932 | if ! wedge-exists mypy $MYPY_VERSION $USER_WEDGE_DIR; then
|
933 |
|
934 | # NOTE: We have to also copy the .git dir, because it has
|
935 | # .git/modules/typeshed
|
936 | local dest_dir=$USER_WEDGE_DIR/pkg/mypy/$MYPY_VERSION
|
937 | mkdir -p $dest_dir
|
938 |
|
939 | # Note: pack files in .git/modules/typeshed/objects/pack are read-only
|
940 | # this can fail
|
941 | cp --verbose --recursive --no-target-directory \
|
942 | $DEPS_SOURCE_DIR/mypy/mypy-$MYPY_VERSION $dest_dir
|
943 | fi
|
944 |
|
945 | if ! wedge-exists py3-libs $PY3_LIBS_VERSION $USER_WEDGE_DIR; then
|
946 | download-py3-libs
|
947 | # This patch doesn't work?
|
948 | # patch-typed-ast
|
949 | install-py3-libs
|
950 | fi
|
951 |
|
952 | if ! wedge-exists souffle $SOUFFLE_VERSION $USER_WEDGE_DIR; then
|
953 | deps/wedge.sh unboxed-build _build/deps-source/souffle/
|
954 | fi
|
955 |
|
956 | if command -v tree > /dev/null; then
|
957 | tree -L 3 $USER_WEDGE_DIR
|
958 | echo
|
959 | tree -L 3 /wedge/oils-for-unix.org
|
960 | fi
|
961 | }
|
962 |
|
963 | install-wedges-py() {
|
964 | install-wedges py_only
|
965 | }
|
966 |
|
967 | #
|
968 | # Unboxed wedge builds
|
969 | #
|
970 |
|
971 | uftrace-host() {
|
972 | ### built on demand; run $0 first
|
973 |
|
974 | # BUG: doesn't detect python3
|
975 | # WEDGE tells me that it depends on pkg-config
|
976 | # 'apt-get install pkgconf' gets it
|
977 | # TODO: Should use python3 WEDGE instead of SYSTEM python3?
|
978 |
|
979 | deps/wedge.sh unboxed-build _build/deps-source/uftrace
|
980 | }
|
981 |
|
982 | bloaty-host() {
|
983 | deps/wedge.sh unboxed-build _build/deps-source/bloaty
|
984 | }
|
985 |
|
986 | R-libs-host() {
|
987 | deps/wedge.sh unboxed-build _build/deps-source/R-libs
|
988 | }
|
989 |
|
990 | #
|
991 | # Wedges built inside a container, for copying into a container
|
992 | #
|
993 |
|
994 | container-wedges() {
|
995 | #### host _build/wedge/binary -> guest container /wedge or ~/wedge
|
996 |
|
997 | #export-podman
|
998 |
|
999 | if true; then
|
1000 | deps/wedge.sh build deps/source.medo/time-helper
|
1001 | deps/wedge.sh build deps/source.medo/cmark/
|
1002 | deps/wedge.sh build deps/source.medo/re2c/
|
1003 | deps/wedge.sh build deps/source.medo/python3/
|
1004 | fi
|
1005 |
|
1006 | if false; then
|
1007 | deps/wedge.sh build deps/source.medo/bloaty/
|
1008 | deps/wedge.sh build deps/source.medo/uftrace/
|
1009 | fi
|
1010 |
|
1011 | if false; then
|
1012 | # For soil-benchmarks/ images
|
1013 | deps/wedge.sh build deps/source.medo/R-libs/
|
1014 | fi
|
1015 | }
|
1016 |
|
1017 | #
|
1018 | # Report
|
1019 | #
|
1020 |
|
1021 | commas() {
|
1022 | # Wow I didn't know this :a trick
|
1023 | #
|
1024 | # OK this is a label and a loop, which makes sense. You can't do it with
|
1025 | # pure regex.
|
1026 | #
|
1027 | # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
|
1028 | # https://shallowsky.com/blog/linux/cmdline/sed-improve-comma-insertion.html
|
1029 | sed ':a;s/\b\([0-9]\+\)\([0-9]\{3\}\)\b/\1,\2/;ta'
|
1030 | }
|
1031 |
|
1032 | wedge-sizes() {
|
1033 | local tmp=_tmp/wedge-sizes.txt
|
1034 |
|
1035 | # -b is --bytes, but use short flag for busybox compat
|
1036 | du -s -b /wedge/*/*/* ~/wedge/*/*/* | awk '
|
1037 | { print $0 # print the line
|
1038 | total_bytes += $1 # accumulate
|
1039 | }
|
1040 | END { print total_bytes " TOTAL" }
|
1041 | ' > $tmp
|
1042 |
|
1043 | # printf justifies du output
|
1044 | cat $tmp | commas | xargs -n 2 printf '%15s %s\n'
|
1045 | echo
|
1046 |
|
1047 | #du -s --si /wedge/*/*/* ~/wedge/*/*/*
|
1048 | #echo
|
1049 | }
|
1050 |
|
1051 | wedge-report() {
|
1052 | # 4 levels deep shows the package
|
1053 | if command -v tree > /dev/null; then
|
1054 | tree -L 4 /wedge ~/wedge
|
1055 | echo
|
1056 | fi
|
1057 |
|
1058 | wedge-sizes
|
1059 |
|
1060 | local tmp=_tmp/wedge-manifest.txt
|
1061 |
|
1062 | echo 'Biggest files'
|
1063 | if ! find /wedge ~/wedge -type f -a -printf '%10s %P\n' > $tmp; then
|
1064 | # busybox find doesn't have -printf
|
1065 | echo 'find -printf failed'
|
1066 | return
|
1067 | fi
|
1068 |
|
1069 | set +o errexit # ignore SIGPIPE
|
1070 | sort -n --reverse $tmp | head -n 20 | commas
|
1071 | set -o errexit
|
1072 |
|
1073 | echo
|
1074 |
|
1075 | # Show the most common file extensions
|
1076 | #
|
1077 | # I feel like we should be able to get rid of .a files? That's 92 MB, second
|
1078 | # most common
|
1079 | #
|
1080 | # There are also duplicate .a files for Python -- should look at how distros
|
1081 | # get rid of those
|
1082 |
|
1083 | cat $tmp | python3 -c '
|
1084 | import os, sys, collections
|
1085 |
|
1086 | bytes = collections.Counter()
|
1087 | files = collections.Counter()
|
1088 |
|
1089 | for line in sys.stdin:
|
1090 | size, path = line.split(None, 1)
|
1091 | path = path.strip() # remove newline
|
1092 | _, ext = os.path.splitext(path)
|
1093 | size = int(size)
|
1094 |
|
1095 | bytes[ext] += size
|
1096 | files[ext] += 1
|
1097 |
|
1098 | #print(bytes)
|
1099 | #print(files)
|
1100 |
|
1101 | n = 20
|
1102 |
|
1103 | print("Most common file types")
|
1104 | for ext, count in files.most_common()[:n]:
|
1105 | print("%10d %s" % (count, ext))
|
1106 |
|
1107 | print()
|
1108 |
|
1109 | print("Total bytes by file type")
|
1110 | for ext, total_bytes in bytes.most_common()[:n]:
|
1111 | print("%10d %s" % (total_bytes, ext))
|
1112 | ' | commas
|
1113 | }
|
1114 |
|
1115 | run-task "$@"
|