OILS / deps / from-apt.sh View on Github | oilshell.org

422 lines, 212 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# deps/from-apt.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# These are needed for bootstrapping pip in Python 3.10
11# (Also used by build/py.sh ubuntu-deps)
12#
13# For building Python 3.10 with working 'pip install'
14# libssl-dev: to download packages
15# libffi-dev: for working setuptools
16# zlib1g-dev: needed for 'import zlib'
17declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
18
19# for deps/from-R.sh
20declare -a R_BUILD_DEPS=(
21 r-base-core # R interpreter
22
23 # ICU for the R stringi package. This makes compilation faster; otherwise
24 # it tries to compile it from source.
25 # https://stringi.gagolewski.com/install.html
26 libicu-dev
27)
28
29install-R() {
30 ### For manual use OUTSIDE container
31 apt-install "${R_BUILD_DEPS[@]}"
32}
33
34# https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
35
36# TODO: Use this for ALL images
37apt-install() {
38 ### Helper to slim down images
39
40 apt-get install -y --no-install-recommends "$@"
41}
42
43init-deb-cache() {
44 rm -f /etc/apt/apt.conf.d/docker-clean
45 echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
46}
47
48readonly -a BUILD_PACKAGES=(
49 gcc
50 g++ # re2c is C++
51 make # to build re2c
52
53 # for cmark
54 cmake
55
56 # cmake -G Ninja can be used
57 ninja-build
58
59 # For 'deps/wedge.sh unboxed-install'
60 sudo
61
62 # uftrace configure uses pkg-config to find python3 flags
63 pkg-config
64
65 # for USDT probes
66 systemtap-sdt-dev
67
68 # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
69 # won't work.
70 # TODO: We should move 'pip install' to build time.
71 "${PY3_BUILD_DEPS[@]}"
72
73 # For installing R packages
74 "${R_BUILD_DEPS[@]}"
75)
76
77layer-wedge-bootstrap-debian() {
78 apt-get update
79
80 # Pass aditional deps
81 apt-install "${BUILD_PACKAGES[@]}" "$@"
82}
83
84layer-wedge-bootstrap-debian-10() {
85 # Default packages
86 layer-wedge-bootstrap-debian
87}
88
89layer-wedge-bootstrap-debian-12() {
90 local -a uftrace_packages=(
91 # uftrace configure detects with #include "Python.h"
92 python3-dev
93 # shared library for uftrace to do dlopen()
94 # requires path in uftrace source
95 libpython3.11
96
97 #libpython3.7
98 )
99
100 layer-wedge-bootstrap-debian "${uftrace_packages[@]}"
101}
102
103layer-python-symlink() {
104 ### A special layer for building CPython; done as root
105 ln -s -f -v /usr/bin/python2 /usr/bin/python
106}
107
108layer-debian-10() {
109 # Can't install packages in Debian without this
110 apt-get update # uses /var/lib/apt
111
112 # uses /var/cache/apt
113 apt-install git python2
114}
115
116layer-debian-12() {
117 # Can't install packages in Debian without this
118 apt-get update # uses /var/lib/apt
119
120 # uses /var/cache/apt
121 # Soil's time-tsv3 can run under python3 too
122 apt-install git python3
123}
124
125layer-locales() {
126 apt-install locales
127 # uncomment in a file
128 sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
129 locale-gen --purge en_US.UTF-8
130}
131
132test-image() {
133 ### For testing builds, not run on CI
134
135 apt-install build-essential "${PY3_BUILD_DEPS[@]}"
136}
137
138wild() {
139 # for build/py.sh all
140 local -a packages=(
141 gcc # 'cc' for compiling Python extensions
142 g++ # for C++ tarball
143
144 python2-dev
145 libreadline-dev
146 curl # wait for cpp-tarball
147 )
148
149 apt-install "${packages[@]}"
150}
151
152dev-minimal() {
153 local -a packages=(
154 # TODO: remove
155 python2-dev # for building Python extensions
156 python-setuptools # Python 2, for flake8
157 python-pip # flake8 typing
158
159 gcc # for building Python extensions
160 libreadline-dev
161
162 python3-setuptools # mypy
163 python3-pip
164 # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
165 # dep, which recently updated to version 1.5.5
166 python3-dev
167
168 # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
169 busybox-static mksh zsh
170
171 gawk
172
173 # 'ps' used by spec tests
174 procps
175 # for oil-spec task
176 jq
177 )
178
179 apt-install "${packages[@]}"
180}
181
182pea() {
183 # For installing MyPy
184 # apt-install python3-pip
185
186 echo 'None'
187}
188
189other-tests() {
190 local -a packages=(
191 libreadline-dev
192 python2-dev # osh2oil needs build/py.sh minimal
193
194 # Compilers for R. TODO: try removing after wedge
195 gcc g++
196
197 make # to build py27.grammar.marshal, ugh
198
199 r-base-core
200 )
201
202 apt-install "${packages[@]}"
203}
204
205cpp-small() {
206 local -a packages=(
207 # for build/py.sh all
208 libreadline-dev
209 python2-dev
210
211 # To compile Oil
212 g++
213 ninja-build
214
215 # For 32-bit binaries with -m32
216 gcc-multilib
217 g++-multilib
218
219 # For some tests
220 gawk
221
222 # for MyPy git clone https://. TODO: remove when the build is hermetic
223 ca-certificates
224
225 # for test/ltrace
226 ltrace
227
228 # for USDT probes
229 systemtap-sdt-dev
230 )
231
232 apt-install "${packages[@]}"
233}
234
235benchmarks() {
236 ### For benchmarks
237
238 local -a packages=(
239 # for build/py.sh all
240 libreadline-dev
241 python2-dev
242
243 # To build Oils
244 g++
245 ninja-build
246 make # to build R packages
247
248 # to create _test/index.html
249 gawk
250
251 # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
252 valgrind
253
254 # benchmarks compare system shells -- they don't use our spec-bin? In case
255 # there are perf bugs caused by the build
256 busybox-static mksh zsh
257
258 # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
259 wget
260 bzip2 # extracting benchmarks/osh-runtime
261 xz-utils
262
263 # For analyzing benchmarks.
264 r-base-core
265
266 # pgrep used by test/stateful in interactive task
267 # TODO: Could move both Python and C++ to their own image
268 # That will be a good use case once we have
269 procps
270 )
271
272 apt-install "${packages[@]}"
273}
274
275bloaty() {
276 local -a packages=(
277 g++ # for C++ tarball
278 curl # wait for cpp-tarball
279 )
280
281 apt-install "${packages[@]}"
282}
283
284benchmarks2() {
285 ### uftrace needs a Python plugin
286
287 local -a packages=(
288 curl # fetch C++ tarball
289 g++ # build it
290
291 # uftrace needs a Python 3 plugin
292 # This is different than 'python3' or 'python3.11' -- it's only the shared
293 # lib?
294 libpython3.11
295
296 # for stable benchmarks.
297 valgrind
298
299 # Analyze uftrace
300 r-base-core
301 )
302
303 apt-install "${packages[@]}"
304}
305
306cpp-spec() {
307 ### For cpp-spec
308
309 local -a packages=(
310 # for build/py.sh all
311 libreadline-dev
312 python2-dev
313
314 # To build Oil
315 g++
316 ninja-build
317
318 # to create _test/index.html
319 gawk
320
321 # spec tests use these
322 procps
323 jq
324
325 # for MyPy git clone https://. TODO: remove when the build is hermetic
326 ca-certificates
327 )
328
329 apt-install "${packages[@]}"
330}
331
332clang() {
333 ### For cpp-coverage
334
335 local -a packages=(
336 # For build/py.sh minimal
337 libreadline-dev
338 python2-dev
339
340 # Compile Oils
341 g++
342 ninja-build
343
344 xz-utils # to extract Clang
345
346 # for MyPy git clone https://. TODO: remove when the build is hermetic
347 ca-certificates
348 )
349
350 apt-install "${packages[@]}"
351}
352
353ovm-tarball() {
354 local -a packages=(
355 # build/py.sh all
356 libreadline-dev
357 python2-dev
358
359 # retrieving spec-bin -- TODO: move to build time
360 wget
361 # for wget https://. TODO: remove when the build is hermetic
362 ca-certificates
363
364 # when spec tests use 'time', dash falls back on 'time' command
365 'time'
366
367 # TODO: probably can remove C++ compiler now that re2c is a wedge
368 gcc
369 g++
370
371 # for cmark
372 cmake
373 # to build Python-2.7.13 (could be a wedge)
374 make
375
376 xz-utils # extract e.g. zsh/yash tarballs
377 bzip2 # extract e.g. busybox tarball
378
379 # for syscall measurements
380 strace
381
382 # used by test/spec-runner.sh
383 gawk
384 )
385
386 apt-install "${packages[@]}"
387}
388
389app-tests() {
390 local -a packages=(
391 # build/py.sh all
392 libreadline-dev
393 python2-dev
394
395 # retrieving spec-bin -- TODO: move to build time
396 wget
397 # for wget https://. TODO: remove when the build is hermetic
398 ca-certificates
399
400 curl # wait for cpp-tarball
401
402 gcc
403 g++ # for C++ tarball
404
405 # to build ble.sh
406 make
407 # used by ble.sh
408 gawk
409 procps
410
411 # for ble.sh contra
412 libx11-dev
413 libxft-dev
414 libncursesw5-dev
415 )
416
417 apt-install "${packages[@]}"
418}
419
420if test $(basename $0) = 'from-apt.sh'; then
421 "$@"
422fi