OILS / deps / wedge.sh View on Github | oilshell.org

419 lines, 205 significant
1#!/usr/bin/env bash
2#
3# Build a wedge.
4#
5# Usage:
6# deps/wedge.sh <function name>
7#
8# Containerized build:
9#
10# $0 build deps/source.medo/re2c/
11#
12# Host build, without containers:
13#
14# $0 unboxed-build deps/source.medo/re2c/
15#
16# Individual steps:
17#
18# $0 unboxed-make deps/source.medo/re2c/
19# $0 unboxed-install deps/source.medo/re2c/
20# $0 unboxed-smoke-test deps/source.medo/re2c/
21#
22# Host dir structure:
23#
24# ~/git/oilshell/oil
25# deps/
26# source.medo/ # Source Files
27# MEDO # points to silo
28# re2c.wedge.sh # later it will be re2c.wedge.hay
29# re2c-3.0.blob # .tar.gz file that you can 'medo sync'
30# re2c-3.1.blob
31# opaque.medo/ # Binary files, e.g. Clang
32# derived.medo/ # Svaed output of 'wedge build'
33#
34# _build/ # Temp dirs and output
35# obj/ # for C++ / Ninja
36# wedge/ # for containerized builds
37# source/ # sync'd from deps/source.medo
38# unboxed-tmp/ # build directory
39# boxed/ # output of containerized build
40# # TODO: rename from /binary/
41
42# Every package ("wedge") has these dirs associated with it:
43#
44# 1. Dir with additional tests / files, near tarball and *.wedge.sh ($wedge_dir)
45# 2. Where it's extracted ($src_dir)
46# 3. The temp dir where you run ./configure --prefix; make; make install ($build_dir)
47# 4. The dir to install to ($install_dir)
48# 5. The temp dir where the smoke test is run
49
50# For Debian/Ubuntu
51
52# Note: xz-utils needed to extract, but medo should make that transparent?
53#
54# Container dir structure
55#
56# /home/uke/
57# tmp-mount/
58# _build/ # Build into this temp dir
59# deps-source/
60# re2c/
61# re2c-3.0.tar.xz
62# re2c-3.0/ # Extract it here
63# wedge/
64# re2c
65# /wedge/ # Output is mounted to oil/_mount/wedge-out
66# oilshell.org/
67# pkg/
68# re2c/
69# 3.0/
70# debug-info/ # Probably needs to be at an absolute path because of
71# # --debug-link
72# re2c/
73# 3.0/
74#
75# Then Dockerfile.wild does:
76#
77# COPY _build/wedge/binary/oils-for-unix.org/pkg/re2c/3.0 \
78# /wedge/oils-for-unix.org/pkg/re2c/3.0
79
80set -o nounset
81set -o pipefail
82set -o errexit
83
84REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
85readonly REPO_ROOT
86
87OILS_ABSOLUTE_ROOT='/wedge/oils-for-unix.org'
88
89# The user may build a wedge outside a container here
90OILS_RELATIVE_ROOT="$HOME/wedge/oils-for-unix.org"
91
92die() {
93 echo "$0: $@" >& 2
94 exit 1
95}
96
97#
98# Dirs
99#
100
101source-dir() {
102 if test -n "${WEDGE_TARBALL_NAME:-}"; then
103
104 # for Python-3.10.4 to override 'python3' package name
105 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_TARBALL_NAME-$WEDGE_VERSION"
106
107 else
108 echo "$REPO_ROOT/_build/deps-source/$WEDGE_NAME/$WEDGE_NAME-$WEDGE_VERSION"
109 fi
110}
111
112build-dir() {
113 # call it tmp-build?
114 echo "$REPO_ROOT/_build/wedge/tmp/$WEDGE_NAME"
115}
116
117install-dir() {
118 local prefix
119 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
120 prefix=$OILS_ABSOLUTE_ROOT
121 else
122 prefix=$OILS_RELATIVE_ROOT
123 fi
124 echo "$prefix/pkg/$WEDGE_NAME/$WEDGE_VERSION"
125}
126
127smoke-test-dir() {
128 echo "$REPO_ROOT/_build/wedge/smoke-test/$WEDGE_NAME"
129}
130
131load-wedge() {
132 ### source .wedge.sh file and ensure it conforms to protocol
133
134 local wedge_dir=$1
135
136 echo "Loading $wedge_dir"
137 echo
138
139 source $wedge_dir/WEDGE
140
141 echo " OK name: ${WEDGE_NAME?"$wedge_dir: WEDGE_NAME required"}"
142 echo " OK version: ${WEDGE_VERSION?"$wedge_dir: WEDGE_VERSION required"}"
143 if test -n "${WEDGE_TARBALL_NAME:-}"; then
144 echo " -- tarball name: $WEDGE_TARBALL_NAME"
145 fi
146 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
147 echo ' -- WEDGE_IS_ABSOLUTE'
148 fi
149
150 # Python and R installation use the network
151 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
152 echo ' -- WEDGE_LEAKY_BUILD'
153 fi
154
155 for func in wedge-make wedge-install wedge-smoke-test; do
156 if declare -f $func > /dev/null; then
157 echo " OK $func"
158 else
159 die "$wedge_dir: $func not declared"
160 fi
161 done
162 echo
163
164 echo "Loaded $wedge_dir"
165 echo
166}
167
168_run-sourced-func() {
169 "$@"
170}
171
172#
173# Actions
174#
175
176validate() {
177 local wedge=$1
178
179 load-wedge $wedge
180}
181
182unboxed-make() {
183 ### Build on the host
184
185 local wedge=$1 # e.g. re2c.wedge.sh
186
187 load-wedge $wedge
188
189 local build_dir=$(build-dir)
190
191 # NOT created because it might require root permissions!
192 local install_dir=$(install-dir)
193
194 rm -r -f -v $build_dir
195 mkdir -p $build_dir
196
197 echo SOURCE $(source-dir)
198
199 # TODO: pushd/popd error handling
200
201 pushd $build_dir
202 wedge-make $(source-dir) $build_dir $install_dir
203 popd
204}
205
206
207# https://www.gnu.org/prep/standards/html_node/Standard-Targets.html
208
209# Do not strip executables when installing them. This helps eventual
210# debugging that may be needed later, and nowadays disk space is cheap and
211# dynamic loaders typically ensure debug sections are not loaded during
212# normal execution. Users that need stripped binaries may invoke the
213# install-strip target to do that.
214
215_unboxed-install() {
216 local wedge=$1 # e.g. re2c.wedge.sh
217
218 load-wedge $wedge
219
220 local install_dir
221 install_dir=$(install-dir)
222 mkdir -p $install_dir
223
224 # Note: install-dir needed for time-helper, but not others
225 wedge-install $(build-dir) $install_dir
226}
227
228unboxed-install() {
229 local wedge=$1 # e.g. re2.wedge.sh
230
231 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
232 sudo $0 _unboxed-install "$@"
233 else
234 _unboxed-install "$@"
235 fi
236}
237
238unboxed-smoke-test() {
239 local wedge_dir=$1 # e.g. re2c/ with WEDGE
240
241 load-wedge $wedge_dir
242
243 local smoke_test_dir=$(smoke-test-dir)
244 local install_dir=$(install-dir)
245
246 echo ' SMOKE TEST'
247
248 local abs_wedge_dir
249 case $wedge_dir in
250 /*) # it's already absolute
251 abs_wedge_dir=$wedge_dir
252 ;;
253 *)
254 abs_wedge_dir=$PWD/$wedge_dir
255 ;;
256 esac
257
258 # TODO: To ensure a clean dir, it might be better to test that it does NOT
259 # exist first, and just make it. If it exists, then remove everything.
260
261 rm -r -f -v $smoke_test_dir
262 mkdir -p $smoke_test_dir
263
264 pushd $smoke_test_dir
265 set -x
266 wedge-smoke-test $install_dir $abs_wedge_dir
267 set +x
268 popd
269
270 echo ' OK'
271}
272
273unboxed-stats() {
274 local wedge=$1
275
276 load-wedge $wedge
277
278 du --si -s $(source-dir)
279 echo
280
281 du --si -s $(build-dir)
282 echo
283
284 du --si -s $(install-dir)
285 echo
286}
287
288unboxed-build() {
289 local wedge_dir=$1
290
291 # TODO:
292 # - Would be nice to export the logs somewhere
293
294 unboxed-make $wedge_dir
295
296 unboxed-install $wedge_dir
297
298 unboxed-smoke-test $wedge_dir
299}
300
301readonly BUILD_IMAGE=oilshell/soil-wedge-builder
302readonly BUILD_IMAGE_TAG=v-2023-03-01
303
304DOCKER=${DOCKER:-docker}
305
306build() {
307 ### Build inside a container, and put output in a specific place.
308
309 # TODO: Specify the container OS, CPU like x86-64, etc.
310
311 local wedge=$1
312
313 load-wedge $wedge
314
315 # Permissions will be different, so we separate the two
316
317 local wedge_host_dir
318 local wedge_guest_dir
319 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
320 wedge_host_dir=_build/wedge/binary # TODO: rename to /absolute/
321 wedge_guest_dir=/wedge
322 else
323 wedge_host_dir=_build/wedge/relative
324 wedge_guest_dir=/home/wedge-builder/wedge
325 fi
326
327 mkdir -v -p $wedge_host_dir
328
329 # TODO:
330 #
331 # Mount
332 # INPUTS: the PKG.wedge.sh, and the tarball
333 # CODE: this script: deps/wedge.sh
334 # OUTPUT: /wedge/oils-for-unix.org
335 # TODO: Also put logs and symbols somewhere
336
337 # Run unboxed-{build,install,smoke-test} INSIDE the container
338 local -a args=(
339 sh -c 'cd ~/oil; deps/wedge.sh unboxed-build $1' dummy "$wedge"
340 )
341
342 local -a docker_flags=()
343 if test -n "${WEDGE_LEAKY_BUILD:-}"; then
344 :
345 else
346 # Disable network for hermetic builds. TODO: Add automated test
347 docker_flags=( --network none )
348 fi
349
350 # TODO:
351 # - Don't mount the whole REPO_ROOT
352 # - We want the bare minimum of files, for cache invalidation
353 # - Maybe make it read only
354 # - Bind mount WEDGE_DEPS='', space separated list of paths
355 # - py3-libs depends on python3 and mypy wedges!
356
357 # -E to preserve CONTAINERS_REGISTRIES_CONF
358 sudo -E $DOCKER run "${docker_flags[@]}" \
359 --mount "type=bind,source=$REPO_ROOT,target=/home/wedge-builder/oil" \
360 --mount "type=bind,source=$PWD/$wedge_host_dir,target=$wedge_guest_dir" \
361 $BUILD_IMAGE:$BUILD_IMAGE_TAG \
362 "${args[@]}"
363}
364
365smoke-test() {
366 local wedge_dir=$1
367 local wedge_out_dir=${2:-_build/wedge/binary} # TODO: rename to /boxed
368 local debug_shell=${3:-}
369
370 load-wedge $wedge_dir
371
372 local -a args=(
373 sh -c 'cd ~/oil; deps/wedge.sh unboxed-smoke-test $1' dummy "$wedge_dir"
374 )
375 local -a docker_flags=()
376 if test -n "$debug_shell"; then
377 docker_flags=( -i -t )
378 args=( "$debug_shell" )
379 fi
380
381 local wedge_mount_dir
382 if test -n "${WEDGE_IS_ABSOLUTE:-}"; then
383 wedge_mount_dir=/wedge
384 else
385 wedge_mount_dir=/home/wedge-builder/wedge
386 fi
387
388 sudo $DOCKER run "${docker_flags[@]}" \
389 --network none \
390 --mount "type=bind,source=$REPO_ROOT,target=/home/wedge-builder/oil" \
391 --mount "type=bind,source=$PWD/$wedge_out_dir,target=$wedge_mount_dir" \
392 $BUILD_IMAGE:$BUILD_IMAGE_TAG \
393 "${args[@]}"
394}
395
396if [[ $# -eq 0 || $1 =~ ^(--help|-h)$ ]]; then
397 # A trick for help. TODO: Move this to a common file, and combine with help
398 # in test/spec.sh.
399
400 awk '
401 $0 ~ /^#/ { print $0 }
402 $0 !~ /^#/ { print ""; exit }
403 ' $0
404 exit
405fi
406
407case $1 in
408 validate|\
409 unboxed-build|\
410 unboxed-make|unboxed-install|_unboxed-install|\
411 unboxed-smoke-test|unboxed-stats|\
412 build|smoke-test)
413 "$@"
414 ;;
415
416 *)
417 die "$0: Invalid action '$1'"
418 ;;
419esac