| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Usage:
|
| 4 | # deps/from-apt.sh <function name>
|
| 5 |
|
| 6 | set -o nounset
|
| 7 | set -o pipefail
|
| 8 | set -o errexit
|
| 9 |
|
| 10 | # These are needed for bootstrapping pip in Python 3.10
|
| 11 | # (Also used by build/py.sh ubuntu-deps)
|
| 12 | #
|
| 13 | # For building Python 3.10 with working 'pip install'
|
| 14 | # libssl-dev: to download packages
|
| 15 | # libffi-dev: for working setuptools
|
| 16 | # zlib1g-dev: needed for 'import zlib'
|
| 17 | declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
|
| 18 |
|
| 19 | # for deps/from-R.sh
|
| 20 | declare -a R_BUILD_DEPS=(
|
| 21 | r-base-core # R interpreter
|
| 22 |
|
| 23 | # ICU for the R stringi package. This makes compilation faster; otherwise
|
| 24 | # it tries to compile it from source.
|
| 25 | # https://stringi.gagolewski.com/install.html
|
| 26 | libicu-dev
|
| 27 | )
|
| 28 |
|
| 29 | install-R() {
|
| 30 | ### For manual use OUTSIDE container
|
| 31 | apt-install "${R_BUILD_DEPS[@]}"
|
| 32 | }
|
| 33 |
|
| 34 | # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
|
| 35 |
|
| 36 | # TODO: Use this for ALL images
|
| 37 | apt-install() {
|
| 38 | ### Helper to slim down images
|
| 39 |
|
| 40 | apt-get install -y --no-install-recommends "$@"
|
| 41 | }
|
| 42 |
|
| 43 | init-deb-cache() {
|
| 44 | rm -f /etc/apt/apt.conf.d/docker-clean
|
| 45 | echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
|
| 46 | }
|
| 47 |
|
| 48 | layer-wedge-builder() {
|
| 49 | local -a packages=(
|
| 50 | # xz-utils # do we need this for extraction?
|
| 51 |
|
| 52 | gcc
|
| 53 | g++ # re2c is C++
|
| 54 | make # to build re2c
|
| 55 |
|
| 56 | # for cmark
|
| 57 | cmake
|
| 58 |
|
| 59 | # cmake -G Ninja can be used
|
| 60 | ninja-build
|
| 61 |
|
| 62 | # For 'deps/wedge.sh unboxed-install'
|
| 63 | sudo
|
| 64 |
|
| 65 | # uftrace configure uses pkg-config to find python3 flags
|
| 66 | pkg-config
|
| 67 | # uftrace configure detects with #include "Python.h"
|
| 68 | python3-dev
|
| 69 | # shared library for uftrace to do dlopen()
|
| 70 | # requires path in uftrace source
|
| 71 | libpython3.7
|
| 72 |
|
| 73 | # for USDT probes
|
| 74 | systemtap-sdt-dev
|
| 75 |
|
| 76 | # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
|
| 77 | # won't work.
|
| 78 | # TODO: We should move 'pip install' to build time.
|
| 79 | "${PY3_BUILD_DEPS[@]}"
|
| 80 |
|
| 81 | # For installing R packages
|
| 82 | "${R_BUILD_DEPS[@]}"
|
| 83 | )
|
| 84 |
|
| 85 | apt-get update
|
| 86 |
|
| 87 | apt-install "${packages[@]}"
|
| 88 | }
|
| 89 |
|
| 90 | layer-python-symlink() {
|
| 91 | ### A special layer for building CPython; done as root
|
| 92 | ln -s -f -v /usr/bin/python2 /usr/bin/python
|
| 93 | }
|
| 94 |
|
| 95 | layer-for-soil() {
|
| 96 | # git: for checking out code
|
| 97 | # python2: for various tools
|
| 98 |
|
| 99 | # TODO: change python2 to python3
|
| 100 | apt-install git python2
|
| 101 | }
|
| 102 |
|
| 103 | layer-common() {
|
| 104 | # with RUN --mount=type=cache
|
| 105 |
|
| 106 | # Can't install packages in Debian without this
|
| 107 | apt-get update # uses /var/lib/apt
|
| 108 |
|
| 109 | layer-for-soil # uses /var/cache/apt
|
| 110 | }
|
| 111 |
|
| 112 | layer-locales() {
|
| 113 | apt-install locales
|
| 114 | # uncomment in a file
|
| 115 | sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
|
| 116 | locale-gen --purge en_US.UTF-8
|
| 117 | }
|
| 118 |
|
| 119 | test-image() {
|
| 120 | ### For testing builds, not run on CI
|
| 121 |
|
| 122 | apt-install build-essential "${PY3_BUILD_DEPS[@]}"
|
| 123 | }
|
| 124 |
|
| 125 | wild() {
|
| 126 | # for build/py.sh all
|
| 127 | local -a packages=(
|
| 128 | gcc # 'cc' for compiling Python extensions
|
| 129 | g++ # for C++ tarball
|
| 130 |
|
| 131 | python2-dev
|
| 132 | libreadline-dev
|
| 133 | curl # wait for cpp-tarball
|
| 134 | )
|
| 135 |
|
| 136 | apt-install "${packages[@]}"
|
| 137 | }
|
| 138 |
|
| 139 | dev-minimal() {
|
| 140 | local -a packages=(
|
| 141 | # TODO: remove
|
| 142 | python2-dev # for building Python extensions
|
| 143 | python-setuptools # Python 2, for flake8
|
| 144 | python-pip # flake8 typing
|
| 145 |
|
| 146 | gcc # for building Python extensions
|
| 147 | libreadline-dev
|
| 148 |
|
| 149 | python3-setuptools # mypy
|
| 150 | python3-pip
|
| 151 | # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
|
| 152 | # dep, which recently updated to version 1.5.5
|
| 153 | python3-dev
|
| 154 |
|
| 155 | # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
|
| 156 | busybox-static mksh zsh
|
| 157 |
|
| 158 | gawk
|
| 159 |
|
| 160 | # 'ps' used by spec tests
|
| 161 | procps
|
| 162 | # for oil-spec task
|
| 163 | jq
|
| 164 | )
|
| 165 |
|
| 166 | apt-install "${packages[@]}"
|
| 167 | }
|
| 168 |
|
| 169 | pea() {
|
| 170 | # For installing MyPy
|
| 171 | # apt-install python3-pip
|
| 172 |
|
| 173 | echo 'None'
|
| 174 | }
|
| 175 |
|
| 176 | other-tests() {
|
| 177 | local -a packages=(
|
| 178 | libreadline-dev
|
| 179 | python2-dev # osh2oil needs build/py.sh minimal
|
| 180 |
|
| 181 | # Compilers for R. TODO: try removing after wedge
|
| 182 | gcc g++
|
| 183 |
|
| 184 | make # to build py27.grammar.marshal, ugh
|
| 185 |
|
| 186 | r-base-core
|
| 187 | )
|
| 188 |
|
| 189 | apt-install "${packages[@]}"
|
| 190 | }
|
| 191 |
|
| 192 | cpp-small() {
|
| 193 | local -a packages=(
|
| 194 | # for build/py.sh all
|
| 195 | libreadline-dev
|
| 196 | python2-dev
|
| 197 |
|
| 198 | # To compile Oil
|
| 199 | g++
|
| 200 | ninja-build
|
| 201 |
|
| 202 | # For 32-bit binaries with -m32
|
| 203 | gcc-multilib
|
| 204 | g++-multilib
|
| 205 |
|
| 206 | # For some tests
|
| 207 | gawk
|
| 208 |
|
| 209 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 210 | ca-certificates
|
| 211 |
|
| 212 | # for test/ltrace
|
| 213 | ltrace
|
| 214 |
|
| 215 | # for USDT probes
|
| 216 | systemtap-sdt-dev
|
| 217 | )
|
| 218 |
|
| 219 | apt-install "${packages[@]}"
|
| 220 | }
|
| 221 |
|
| 222 | benchmarks() {
|
| 223 | ### For benchmarks
|
| 224 |
|
| 225 | local -a packages=(
|
| 226 | # for build/py.sh all
|
| 227 | libreadline-dev
|
| 228 | python2-dev
|
| 229 |
|
| 230 | # To build Oil
|
| 231 | g++
|
| 232 | ninja-build
|
| 233 | make # to build R packages
|
| 234 |
|
| 235 | # to create _test/index.html
|
| 236 | gawk
|
| 237 |
|
| 238 | # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
|
| 239 | valgrind
|
| 240 |
|
| 241 | # benchmarks compare system shells -- they don't use our spec-bin? In case
|
| 242 | # there are perf bugs caused by the build
|
| 243 | busybox-static mksh zsh
|
| 244 |
|
| 245 | # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
|
| 246 | wget
|
| 247 | bzip2 # extracting benchmarks/osh-runtime
|
| 248 | xz-utils
|
| 249 |
|
| 250 | # For analyzing benchmarks.
|
| 251 | r-base-core
|
| 252 |
|
| 253 | # pgrep used by test/stateful in interactive task
|
| 254 | # TODO: Could move both Python and C++ to their own image
|
| 255 | # That will be a good use case once we have
|
| 256 | procps
|
| 257 | )
|
| 258 |
|
| 259 | apt-install "${packages[@]}"
|
| 260 | }
|
| 261 |
|
| 262 | benchmarks2() {
|
| 263 | ### uftrace needs a Python plugin
|
| 264 |
|
| 265 | local -a packages=(
|
| 266 | # for build/py.sh all
|
| 267 | libreadline-dev
|
| 268 | python2-dev
|
| 269 |
|
| 270 | # To build Oil
|
| 271 | g++
|
| 272 | ninja-build
|
| 273 |
|
| 274 | # uftrace needs a Python 3 plugin
|
| 275 | # Technically we don't need 'python3' or 'python3.7' -- only the shared
|
| 276 | # lib?
|
| 277 | libpython3.7
|
| 278 |
|
| 279 | # for stable benchmarks.
|
| 280 | valgrind
|
| 281 |
|
| 282 | # Analyze uftrace
|
| 283 | r-base-core
|
| 284 |
|
| 285 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 286 | ca-certificates
|
| 287 | )
|
| 288 |
|
| 289 | apt-install "${packages[@]}"
|
| 290 | }
|
| 291 |
|
| 292 | cpp-spec() {
|
| 293 | ### For cpp-spec
|
| 294 |
|
| 295 | local -a packages=(
|
| 296 | # for build/py.sh all
|
| 297 | libreadline-dev
|
| 298 | python2-dev
|
| 299 |
|
| 300 | # To build Oil
|
| 301 | g++
|
| 302 | ninja-build
|
| 303 |
|
| 304 | # to create _test/index.html
|
| 305 | gawk
|
| 306 |
|
| 307 | # spec tests use these
|
| 308 | procps
|
| 309 | jq
|
| 310 |
|
| 311 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 312 | ca-certificates
|
| 313 | )
|
| 314 |
|
| 315 | apt-install "${packages[@]}"
|
| 316 | }
|
| 317 |
|
| 318 | clang() {
|
| 319 | ### For cpp-coverage
|
| 320 |
|
| 321 | local -a packages=(
|
| 322 | # For build/py.sh minimal
|
| 323 | libreadline-dev
|
| 324 | python2-dev
|
| 325 |
|
| 326 | # Compile Oils
|
| 327 | g++
|
| 328 | ninja-build
|
| 329 |
|
| 330 | xz-utils # to extract Clang
|
| 331 |
|
| 332 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 333 | ca-certificates
|
| 334 | )
|
| 335 |
|
| 336 | apt-install "${packages[@]}"
|
| 337 | }
|
| 338 |
|
| 339 | ovm-tarball() {
|
| 340 | local -a packages=(
|
| 341 | # build/py.sh all
|
| 342 | libreadline-dev
|
| 343 | python2-dev
|
| 344 |
|
| 345 | # retrieving spec-bin -- TODO: move to build time
|
| 346 | wget
|
| 347 | # for wget https://. TODO: remove when the build is hermetic
|
| 348 | ca-certificates
|
| 349 |
|
| 350 | # when spec tests use 'time', dash falls back on 'time' command
|
| 351 | 'time'
|
| 352 |
|
| 353 | # TODO: probably can remove C++ compiler now that re2c is a wedge
|
| 354 | gcc
|
| 355 | g++
|
| 356 |
|
| 357 | # for cmark
|
| 358 | cmake
|
| 359 | # to build Python-2.7.13 (could be a wedge)
|
| 360 | make
|
| 361 |
|
| 362 | xz-utils # extract e.g. zsh/yash tarballs
|
| 363 | bzip2 # extract e.g. busybox tarball
|
| 364 |
|
| 365 | # for syscall measurements
|
| 366 | strace
|
| 367 |
|
| 368 | # used by test/spec-runner.sh
|
| 369 | gawk
|
| 370 | )
|
| 371 |
|
| 372 | apt-install "${packages[@]}"
|
| 373 | }
|
| 374 |
|
| 375 | app-tests() {
|
| 376 | local -a packages=(
|
| 377 | # build/py.sh all
|
| 378 | libreadline-dev
|
| 379 | python2-dev
|
| 380 |
|
| 381 | # retrieving spec-bin -- TODO: move to build time
|
| 382 | wget
|
| 383 | # for wget https://. TODO: remove when the build is hermetic
|
| 384 | ca-certificates
|
| 385 |
|
| 386 | curl # wait for cpp-tarball
|
| 387 |
|
| 388 | gcc
|
| 389 | g++ # for C++ tarball
|
| 390 |
|
| 391 | # to build ble.sh
|
| 392 | make
|
| 393 | # used by ble.sh
|
| 394 | gawk
|
| 395 | procps
|
| 396 |
|
| 397 | # for ble.sh contra
|
| 398 | libx11-dev
|
| 399 | libxft-dev
|
| 400 | libncursesw5-dev
|
| 401 | )
|
| 402 |
|
| 403 | apt-install "${packages[@]}"
|
| 404 | }
|
| 405 |
|
| 406 | if test $(basename $0) = 'from-apt.sh'; then
|
| 407 | "$@"
|
| 408 | fi
|