OILS / deps / wedge.sh View on Github | oilshell.org

473 lines, 238 significant
1#!/usr/bin/env bash
2#
3# Build a wedge.
4#
5# Usage:
6# deps/wedge.sh <function name>
7#
8# Containerized build:
9#
10# $0 build deps/source.medo/re2c/
11#
12# Host build, without containers:
13#
14# $0 unboxed deps/source.medo/re2c/
15#
16# Individual steps:
17#
18# $0 unboxed-make deps/source.medo/re2c/
19# $0 unboxed-install deps/source.medo/re2c/
20# $0 unboxed-smoke-test deps/source.medo/re2c/
21#
22# Host dir structure:
23#
24# ~/git/oilshell/oil
25# deps/
26# source.medo/ # Source Files
27# MEDO # points to silo
28# re2c.wedge.sh # later it will be re2c.wedge.hay
29# re2c-3.0.blob # .tar.gz file that you can 'medo sync'
30# re2c-3.1.blob
31# opaque.medo/ # Binary files, e.g. Clang
32# derived.medo/ # Saved output of 'wedge build'
33#
34# _build/ # Temp dirs and output
35# obj/ # for C++ / Ninja
36# deps-source/ # sync'd from deps/source.medo - should it be
37# # _build/wedge/source?
38# wedge/ # for containerized builds
39# tmp/ # build directory
40# boxed/ # output of containerized build
41# # TODO: rename from /binary/
42# logs/
43# smoke-test/ # current dir for smoke test
44
45# Every package ("wedge") has these dirs associated with it:
46#
47# 1. Dir with additional tests / files, near tarball and *.wedge.sh ($wedge_dir)
48# 2. Where it's extracted ($src_dir)
49# 3. The temp dir where you run ./configure --prefix; make; make install ($build_dir)
50# 4. The dir to install to ($install_dir)
51# 5. The temp dir where the smoke test is run
52
53# For Debian/Ubuntu
54
55# Note: xz-utils needed to extract, but medo should make that transparent?
56#
57# Container dir structure
58#
59# /home/uke/
60# tmp-mount/
61# _build/ # Build into this temp dir
62# deps-source/
63# re2c/
64# re2c-3.0.tar.xz
65# re2c-3.0/ # Extract it here
66# wedge/
67# re2c
68# /wedge/ # Output is mounted to oil/_mount/wedge-out
69# oilshell.org/
70# pkg/
71# re2c/
72# 3.0/
73# debug-info/ # Probably needs to be at an absolute path because of
74# # --debug-link
75# re2c/
76# 3.0/
77#
78# Then Dockerfile.wild does:
79#
80# COPY _build/wedge/binary/oils-for-unix.org/pkg/re2c/3.0 \
81# /wedge/oils-for-unix.org/pkg/re2c/3.0
82
83set -o nounset
84set -o pipefail
85set -o errexit
86
87REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
88readonly REPO_ROOT
89
90OILS_ABSOLUTE_ROOT='/wedge/oils-for-unix.org'
91
92# The user may build a wedge outside a container here
93OILS_RELATIVE_ROOT="$HOME/wedge/oils-for-unix.org"
94
95die() {
96 echo "$0: $@" >& 2
97 exit 1
98}
99
100#
101# Dirs
102#
103
104source-dir() {
105 if test -n "${WEDGE_TARBALL_NAME:-}"; then
106 # for Python-3.10.4 to override 'python3' package name
107 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_TARBALL_NAME-$WEDGE_VERSION"
108
109 else
110 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_NAME-$WEDGE_VERSION"
111 fi
112}
113
114build-dir() {
115 # call it tmp-build?
116 echo "$REPO_ROOT/_build/wedge/tmp/$WEDGE_NAME-$WEDGE_VERSION"
117}
118
119install-dir() {
120 local prefix
121 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
122 prefix=$OILS_ABSOLUTE_ROOT
123 else
124 prefix=$OILS_RELATIVE_ROOT
125 fi
126
127 # TODO: We want to support multiple versions of the same wedge
128 # So maybe we can provide
129 #
130 # WEDGE_VERSION_LIST='4.4 5.2'
131 #
132 # And then provide a flag to select them?
133
134 echo "$prefix/pkg/$WEDGE_NAME/$WEDGE_VERSION"
135}
136
137smoke-test-dir() {
138 echo "$REPO_ROOT/_build/wedge/smoke-test/$WEDGE_NAME-$WEDGE_VERSION"
139}
140
141load-wedge() {
142 ### source .wedge.sh file and ensure it conforms to protocol
143
144 local wedge_dir=$1
145 local version_requested=${2:-}
146
147 echo "Loading $wedge_dir"
148 echo
149
150 source $wedge_dir/WEDGE
151
152 echo " OK name: ${WEDGE_NAME?"$wedge_dir: WEDGE_NAME required"}"
153
154 # This WEDGE supports a single version.
155 if test -n "${WEDGE_VERSION:-}"; then
156 echo " -- single version: $WEDGE_VERSION"
157 fi
158
159 # Can validate version against this
160 if test -n "${WEDGE_VERSION_LIST:-}"; then
161 echo " -- version list: $WEDGE_VERSION_LIST"
162
163 if test -z "$version_requested"; then
164 die "FAIL Expected explicit version, one of: $WEDGE_VERSION_LIST"
165 fi
166
167 case "$WEDGE_VERSION_LIST" in
168 *"$version_requested"*)
169 echo " OK Setting WEDGE_VERSION to $version_requested"
170 WEDGE_VERSION=$version_requested
171 ;;
172 *)
173 die "FAIL Requested version $version_requested should be one of: $WEDGE_VERSION_LIST"
174 ;;
175 esac
176 fi
177
178 if test -n "${WEDGE_TARBALL_NAME:-}"; then
179 echo " -- tarball name: $WEDGE_TARBALL_NAME"
180 fi
181 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
182 echo ' -- WEDGE_IS_ABSOLUTE'
183 fi
184
185 # Python and R installation use the network
186 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
187 echo ' -- WEDGE_LEAKY_BUILD'
188 fi
189
190 for func in wedge-make wedge-install wedge-smoke-test; do
191 if declare -f $func > /dev/null; then
192 echo " OK $func"
193 else
194 die "$wedge_dir: $func not declared"
195 fi
196 done
197 echo
198
199 echo "Loaded $wedge_dir"
200 echo
201}
202
203_run-sourced-func() {
204 "$@"
205}
206
207#
208# Actions
209#
210
211validate() {
212 local wedge=$1
213 local version_requested=${2:-}
214
215 load-wedge $wedge "$version_requested"
216}
217
218unboxed-make() {
219 ### Build on the host
220
221 local wedge=$1 # e.g. re2c.wedge.sh
222 local version_requested=${2:-} # e.g. 5.2
223
224 load-wedge $wedge "$version_requested"
225
226 local source_dir
227 source_dir=$(source-dir)
228 echo " SRC $source_dir"
229
230 local build_dir
231 build_dir=$(build-dir)
232
233 # NOT created because it might require root permissions!
234 local install_dir
235 install_dir=$(install-dir)
236
237 rm -r -f -v $build_dir
238 mkdir -p $build_dir
239
240 # TODO: pushd/popd error handling
241
242 pushd $build_dir
243 wedge-make $source_dir $build_dir $install_dir
244 popd
245}
246
247
248# https://www.gnu.org/prep/standards/html_node/Standard-Targets.html
249
250# Do not strip executables when installing them. This helps eventual
251# debugging that may be needed later, and nowadays disk space is cheap and
252# dynamic loaders typically ensure debug sections are not loaded during
253# normal execution. Users that need stripped binaries may invoke the
254# install-strip target to do that.
255
256_unboxed-install() {
257 local wedge=$1 # e.g. re2c.wedge.sh
258 local version_requested=${2:-} # e.g. 5.2
259
260 load-wedge $wedge "$version_requested"
261
262 local build_dir
263 build_dir=$(build-dir)
264
265 local install_dir
266 install_dir=$(install-dir)
267 mkdir -p $install_dir
268
269 # Note: install-dir needed for time-helper, but not others
270 wedge-install $build_dir $install_dir
271}
272
273unboxed-install() {
274 local wedge=$1 # e.g. re2.wedge.sh
275
276 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
277 sudo $0 _unboxed-install "$@"
278 else
279 _unboxed-install "$@"
280 fi
281}
282
283unboxed-smoke-test() {
284 local wedge_dir=$1 # e.g. re2c/ with WEDGE
285 local version_requested=${2:-} # e.g. 5.2
286
287 load-wedge $wedge_dir "$version_requested"
288
289 local smoke_test_dir
290 smoke_test_dir=$(smoke-test-dir)
291 local install_dir
292 install_dir=$(install-dir)
293
294 echo ' SMOKE TEST'
295
296 local abs_wedge_dir
297 case $wedge_dir in
298 /*) # it's already absolute
299 abs_wedge_dir=$wedge_dir
300 ;;
301 *)
302 abs_wedge_dir=$PWD/$wedge_dir
303 ;;
304 esac
305
306 # TODO: To ensure a clean dir, it might be better to test that it does NOT
307 # exist first, and just make it. If it exists, then remove everything.
308
309 rm -r -f -v $smoke_test_dir
310 mkdir -p $smoke_test_dir
311
312 pushd $smoke_test_dir
313 set -x
314 wedge-smoke-test $install_dir $abs_wedge_dir
315 set +x
316 popd
317
318 echo ' OK'
319}
320
321unboxed-stats() {
322 local wedge=$1
323
324 load-wedge $wedge "$version_requested"
325
326 du --si -s $(source-dir)
327 echo
328
329 du --si -s $(build-dir)
330 echo
331
332 du --si -s $(install-dir)
333 echo
334}
335
336unboxed() {
337 local wedge_dir=$1
338
339 # Can override default version. Could be a flag since it's optional? But
340 # right now we always pass it.
341 local version_requested=${2:-}
342
343 # TODO:
344 # - Would be nice to export the logs somewhere
345
346 unboxed-make $wedge_dir "$version_requested"
347
348 unboxed-install $wedge_dir "$version_requested"
349
350 unboxed-smoke-test $wedge_dir "$version_requested"
351}
352
353readonly BUILD_IMAGE=oilshell/soil-wedge-builder
354readonly BUILD_IMAGE_TAG=v-2023-03-01
355
356DOCKER=${DOCKER:-docker}
357
358boxed() {
359 ### Build inside a container, and put output in a specific place.
360
361 # TODO: Specify the container OS, CPU like x86-64, etc.
362
363 local wedge=$1
364 local version_requested=${2:-}
365
366 load-wedge $wedge "$version_requested"
367
368 # Permissions will be different, so we separate the two
369
370 local wedge_host_dir
371 local wedge_guest_dir
372 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
373 wedge_host_dir=_build/wedge/binary # TODO: rename to /absolute/
374 wedge_guest_dir=/wedge
375 else
376 wedge_host_dir=_build/wedge/relative
377 wedge_guest_dir=/home/wedge-builder/wedge
378 fi
379
380 mkdir -v -p $wedge_host_dir
381
382 # TODO:
383 #
384 # Mount
385 # INPUTS: the PKG.wedge.sh, and the tarball
386 # CODE: this script: deps/wedge.sh
387 # OUTPUT: /wedge/oils-for-unix.org
388 # TODO: Also put logs and symbols somewhere
389
390 # Run unboxed-{build,install,smoke-test} INSIDE the container
391 local -a args=(
392 sh -c 'cd ~/oil; deps/wedge.sh unboxed $1' dummy "$wedge"
393 )
394
395 local -a docker_flags=()
396 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
397 :
398 else
399 # Disable network for hermetic builds. TODO: Add automated test
400 docker_flags=( --network none )
401 fi
402
403 # TODO:
404 # - Don't mount the whole REPO_ROOT
405 # - We want the bare minimum of files, for cache invalidation
406 # - Maybe make it read only
407 # - Bind mount WEDGE_DEPS='', space separated list of paths
408 # - py3-libs depends on python3 and mypy wedges!
409
410 # -E to preserve CONTAINERS_REGISTRIES_CONF
411 sudo -E $DOCKER run "${docker_flags[@]}" \
412 --mount "type=bind,source=$REPO_ROOT,target=/home/wedge-builder/oil" \
413 --mount "type=bind,source=$PWD/$wedge_host_dir,target=$wedge_guest_dir" \
414 $BUILD_IMAGE:$BUILD_IMAGE_TAG \
415 "${args[@]}"
416}
417
418smoke-test() {
419 local wedge_dir=$1
420 local wedge_out_dir=${2:-_build/wedge/binary} # TODO: rename to /boxed
421 local version_requested=${3:-}
422 local debug_shell=${4:-}
423
424 load-wedge $wedge_dir "$version_requested"
425
426 local -a args=(
427 sh -c 'cd ~/oil; deps/wedge.sh unboxed-smoke-test $1' dummy "$wedge_dir"
428 )
429 local -a docker_flags=()
430 if test -n "$debug_shell"; then
431 docker_flags=( -i -t )
432 args=( "$debug_shell" )
433 fi
434
435 local wedge_mount_dir
436 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
437 wedge_mount_dir=/wedge
438 else
439 wedge_mount_dir=/home/wedge-builder/wedge
440 fi
441
442 sudo $DOCKER run "${docker_flags[@]}" \
443 --network none \
444 --mount "type=bind,source=$REPO_ROOT,target=/home/wedge-builder/oil" \
445 --mount "type=bind,source=$PWD/$wedge_out_dir,target=$wedge_mount_dir" \
446 $BUILD_IMAGE:$BUILD_IMAGE_TAG \
447 "${args[@]}"
448}
449
450if [[ $# -eq 0 || $1 =~ ^(--help|-h)$ ]]; then
451 # A trick for help. TODO: Move this to a common file, and combine with help
452 # in test/spec.sh.
453
454 awk '
455 $0 ~ /^#/ { print $0 }
456 $0 !~ /^#/ { print ""; exit }
457 ' $0
458 exit
459fi
460
461case $1 in
462 validate|\
463 unboxed|\
464 unboxed-make|unboxed-install|_unboxed-install|\
465 unboxed-smoke-test|unboxed-stats|\
466 boxed|smoke-test)
467 "$@"
468 ;;
469
470 *)
471 die "$0: Invalid action '$1'"
472 ;;
473esac