| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Usage:
|
| 4 | # deps/from-apt.sh <function name>
|
| 5 |
|
| 6 | set -o nounset
|
| 7 | set -o pipefail
|
| 8 | set -o errexit
|
| 9 |
|
| 10 | # These are needed for bootstrapping pip in Python 3.10
|
| 11 | # (Also used by build/py.sh ubuntu-deps)
|
| 12 | #
|
| 13 | # For building Python 3.10 with working 'pip install'
|
| 14 | # libssl-dev: to download packages
|
| 15 | # libffi-dev: for working setuptools
|
| 16 | # zlib1g-dev: needed for 'import zlib'
|
| 17 | declare -a PY3_BUILD_DEPS=(libssl-dev libffi-dev zlib1g-dev)
|
| 18 |
|
| 19 | # for deps/from-R.sh
|
| 20 | declare -a R_BUILD_DEPS=(
|
| 21 | r-base-core # R interpreter
|
| 22 |
|
| 23 | # ICU for the R stringi package. This makes compilation faster; otherwise
|
| 24 | # it tries to compile it from source.
|
| 25 | # https://stringi.gagolewski.com/install.html
|
| 26 | libicu-dev
|
| 27 | )
|
| 28 |
|
| 29 | install-R() {
|
| 30 | ### For manual use OUTSIDE container
|
| 31 | apt-install "${R_BUILD_DEPS[@]}"
|
| 32 | }
|
| 33 |
|
| 34 | # https://github.com/moby/buildkit/blob/master/frontend/dockerfile/docs/reference.md#run---mounttypecache
|
| 35 |
|
| 36 | # TODO: Use this for ALL images
|
| 37 | apt-install() {
|
| 38 | ### Helper to slim down images
|
| 39 |
|
| 40 | apt-get install -y --no-install-recommends "$@"
|
| 41 | }
|
| 42 |
|
| 43 | init-deb-cache() {
|
| 44 | rm -f /etc/apt/apt.conf.d/docker-clean
|
| 45 | echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
|
| 46 | }
|
| 47 |
|
| 48 | layer-wedge-builder() {
|
| 49 | local -a packages=(
|
| 50 | # xz-utils # do we need this for extraction?
|
| 51 |
|
| 52 | gcc
|
| 53 | g++ # re2c is C++
|
| 54 | make # to build re2c
|
| 55 |
|
| 56 | # for cmark
|
| 57 | cmake
|
| 58 |
|
| 59 | # cmake -G Ninja can be used
|
| 60 | ninja-build
|
| 61 |
|
| 62 | # For 'deps/wedge.sh unboxed-install'
|
| 63 | sudo
|
| 64 |
|
| 65 | # uftrace configure uses pkg-config to find python3 flags
|
| 66 | pkg-config
|
| 67 | # uftrace configure detects with #include "Python.h"
|
| 68 | python3-dev
|
| 69 | # shared library for uftrace to do dlopen()
|
| 70 | # requires path in uftrace source
|
| 71 | libpython3.7
|
| 72 |
|
| 73 | # Dependencies for building our own Python3 wedge. Otherwise 'pip install'
|
| 74 | # won't work.
|
| 75 | # TODO: We should move 'pip install' to build time.
|
| 76 | "${PY3_BUILD_DEPS[@]}"
|
| 77 |
|
| 78 | # For installing R packages
|
| 79 | "${R_BUILD_DEPS[@]}"
|
| 80 | )
|
| 81 |
|
| 82 | apt-get update
|
| 83 |
|
| 84 | apt-install "${packages[@]}"
|
| 85 | }
|
| 86 |
|
| 87 | layer-python-symlink() {
|
| 88 | ### A special layer for building CPython; done as root
|
| 89 | ln -s -f -v /usr/bin/python2 /usr/bin/python
|
| 90 | }
|
| 91 |
|
| 92 | layer-for-soil() {
|
| 93 | # git: for checking out code
|
| 94 | # python2: for various tools
|
| 95 |
|
| 96 | # TODO: change python2 to python3
|
| 97 | apt-install git python2
|
| 98 | }
|
| 99 |
|
| 100 | layer-common() {
|
| 101 | # with RUN --mount=type=cache
|
| 102 |
|
| 103 | # Can't install packages in Debian without this
|
| 104 | apt-get update # uses /var/lib/apt
|
| 105 |
|
| 106 | layer-for-soil # uses /var/cache/apt
|
| 107 | }
|
| 108 |
|
| 109 | layer-locales() {
|
| 110 | apt-install locales
|
| 111 | # uncomment in a file
|
| 112 | sed -i 's/# en_US.UTF-8/en_US.UTF-8/' /etc/locale.gen
|
| 113 | locale-gen --purge en_US.UTF-8
|
| 114 | }
|
| 115 |
|
| 116 | test-image() {
|
| 117 | ### For testing builds, not run on CI
|
| 118 |
|
| 119 | apt-install build-essential "${PY3_BUILD_DEPS[@]}"
|
| 120 | }
|
| 121 |
|
| 122 | wild() {
|
| 123 | # for build/py.sh all
|
| 124 | local -a packages=(
|
| 125 | gcc # 'cc' for compiling Python extensions
|
| 126 | g++ # for C++ tarball
|
| 127 |
|
| 128 | python2-dev
|
| 129 | libreadline-dev
|
| 130 | curl # wait for cpp-tarball
|
| 131 | )
|
| 132 |
|
| 133 | apt-install "${packages[@]}"
|
| 134 | }
|
| 135 |
|
| 136 | dev-minimal() {
|
| 137 | local -a packages=(
|
| 138 | # TODO: remove
|
| 139 | python2-dev # for building Python extensions
|
| 140 | python-setuptools # Python 2, for flake8
|
| 141 | python-pip # flake8 typing
|
| 142 |
|
| 143 | gcc # for building Python extensions
|
| 144 | libreadline-dev
|
| 145 |
|
| 146 | python3-setuptools # mypy
|
| 147 | python3-pip
|
| 148 | # 2023-07: somehow this became necessary to pip3 install typed_ast, a MyPy
|
| 149 | # dep, which recently updated to version 1.5.5
|
| 150 | python3-dev
|
| 151 |
|
| 152 | # Note: osh-minimal task needs shells; testing WITHOUT spec-bin shells
|
| 153 | busybox-static mksh zsh
|
| 154 |
|
| 155 | gawk
|
| 156 |
|
| 157 | # 'ps' used by spec tests
|
| 158 | procps
|
| 159 | # for oil-spec task
|
| 160 | jq
|
| 161 | )
|
| 162 |
|
| 163 | apt-install "${packages[@]}"
|
| 164 | }
|
| 165 |
|
| 166 | pea() {
|
| 167 | # For installing MyPy
|
| 168 | # apt-install python3-pip
|
| 169 |
|
| 170 | echo 'None'
|
| 171 | }
|
| 172 |
|
| 173 | other-tests() {
|
| 174 | local -a packages=(
|
| 175 | libreadline-dev
|
| 176 | python2-dev # osh2oil needs build/py.sh minimal
|
| 177 |
|
| 178 | # Compilers for R. TODO: try removing after wedge
|
| 179 | gcc g++
|
| 180 |
|
| 181 | make # to build py27.grammar.marshal, ugh
|
| 182 |
|
| 183 | r-base-core
|
| 184 | )
|
| 185 |
|
| 186 | apt-install "${packages[@]}"
|
| 187 | }
|
| 188 |
|
| 189 | cpp-small() {
|
| 190 | local -a packages=(
|
| 191 | # for build/py.sh all
|
| 192 | libreadline-dev
|
| 193 | python2-dev
|
| 194 |
|
| 195 | # To compile Oil
|
| 196 | g++
|
| 197 | ninja-build
|
| 198 |
|
| 199 | # For 32-bit binaries with -m32
|
| 200 | gcc-multilib
|
| 201 | g++-multilib
|
| 202 |
|
| 203 | # For some tests
|
| 204 | gawk
|
| 205 |
|
| 206 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 207 | ca-certificates
|
| 208 |
|
| 209 | # for test/ltrace
|
| 210 | ltrace
|
| 211 | )
|
| 212 |
|
| 213 | apt-install "${packages[@]}"
|
| 214 | }
|
| 215 |
|
| 216 | benchmarks() {
|
| 217 | ### For benchmarks
|
| 218 |
|
| 219 | local -a packages=(
|
| 220 | # for build/py.sh all
|
| 221 | libreadline-dev
|
| 222 | python2-dev
|
| 223 |
|
| 224 | # To build Oil
|
| 225 | g++
|
| 226 | ninja-build
|
| 227 | make # to build R packages
|
| 228 |
|
| 229 | # to create _test/index.html
|
| 230 | gawk
|
| 231 |
|
| 232 | # for stable benchmarks. TODO: could move osh-parser cachegrind to benchmarks2
|
| 233 | valgrind
|
| 234 |
|
| 235 | # benchmarks compare system shells -- they don't use our spec-bin? In case
|
| 236 | # there are perf bugs caused by the build
|
| 237 | busybox-static mksh zsh
|
| 238 |
|
| 239 | # retrieving deps like benchmarks/osh-runtime -- TODO: move to build time
|
| 240 | wget
|
| 241 | bzip2 # extracting benchmarks/osh-runtime
|
| 242 | xz-utils
|
| 243 |
|
| 244 | # For analyzing benchmarks.
|
| 245 | r-base-core
|
| 246 |
|
| 247 | # pgrep used by test/stateful in interactive task
|
| 248 | # TODO: Could move both Python and C++ to their own image
|
| 249 | # That will be a good use case once we have
|
| 250 | procps
|
| 251 | )
|
| 252 |
|
| 253 | apt-install "${packages[@]}"
|
| 254 | }
|
| 255 |
|
| 256 | benchmarks2() {
|
| 257 | ### uftrace needs a Python plugin
|
| 258 |
|
| 259 | local -a packages=(
|
| 260 | # for build/py.sh all
|
| 261 | libreadline-dev
|
| 262 | python2-dev
|
| 263 |
|
| 264 | # To build Oil
|
| 265 | g++
|
| 266 | ninja-build
|
| 267 |
|
| 268 | # uftrace needs a Python 3 plugin
|
| 269 | # Technically we don't need 'python3' or 'python3.7' -- only the shared
|
| 270 | # lib?
|
| 271 | libpython3.7
|
| 272 |
|
| 273 | # for stable benchmarks.
|
| 274 | valgrind
|
| 275 |
|
| 276 | # Analyze uftrace
|
| 277 | r-base-core
|
| 278 |
|
| 279 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 280 | ca-certificates
|
| 281 | )
|
| 282 |
|
| 283 | apt-install "${packages[@]}"
|
| 284 | }
|
| 285 |
|
| 286 | cpp-spec() {
|
| 287 | ### For cpp-spec
|
| 288 |
|
| 289 | local -a packages=(
|
| 290 | # for build/py.sh all
|
| 291 | libreadline-dev
|
| 292 | python2-dev
|
| 293 |
|
| 294 | # To build Oil
|
| 295 | g++
|
| 296 | ninja-build
|
| 297 |
|
| 298 | # to create _test/index.html
|
| 299 | gawk
|
| 300 |
|
| 301 | # spec tests use these
|
| 302 | procps
|
| 303 | jq
|
| 304 |
|
| 305 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 306 | ca-certificates
|
| 307 | )
|
| 308 |
|
| 309 | apt-install "${packages[@]}"
|
| 310 | }
|
| 311 |
|
| 312 | clang() {
|
| 313 | ### For cpp-coverage
|
| 314 |
|
| 315 | local -a packages=(
|
| 316 | # For build/py.sh minimal
|
| 317 | libreadline-dev
|
| 318 | python2-dev
|
| 319 |
|
| 320 | # Compile Oils
|
| 321 | g++
|
| 322 | ninja-build
|
| 323 |
|
| 324 | xz-utils # to extract Clang
|
| 325 |
|
| 326 | # for MyPy git clone https://. TODO: remove when the build is hermetic
|
| 327 | ca-certificates
|
| 328 | )
|
| 329 |
|
| 330 | apt-install "${packages[@]}"
|
| 331 | }
|
| 332 |
|
| 333 | ovm-tarball() {
|
| 334 | local -a packages=(
|
| 335 | # build/py.sh all
|
| 336 | libreadline-dev
|
| 337 | python2-dev
|
| 338 |
|
| 339 | # retrieving spec-bin -- TODO: move to build time
|
| 340 | wget
|
| 341 | # for wget https://. TODO: remove when the build is hermetic
|
| 342 | ca-certificates
|
| 343 |
|
| 344 | # when spec tests use 'time', dash falls back on 'time' command
|
| 345 | 'time'
|
| 346 |
|
| 347 | # TODO: probably can remove C++ compiler now that re2c is a wedge
|
| 348 | gcc
|
| 349 | g++
|
| 350 |
|
| 351 | # for cmark
|
| 352 | cmake
|
| 353 | # to build Python-2.7.13 (could be a wedge)
|
| 354 | make
|
| 355 |
|
| 356 | xz-utils # extract e.g. zsh/yash tarballs
|
| 357 | bzip2 # extract e.g. busybox tarball
|
| 358 |
|
| 359 | # for syscall measurements
|
| 360 | strace
|
| 361 |
|
| 362 | # used by test/spec-runner.sh
|
| 363 | gawk
|
| 364 | )
|
| 365 |
|
| 366 | apt-install "${packages[@]}"
|
| 367 | }
|
| 368 |
|
| 369 | app-tests() {
|
| 370 | local -a packages=(
|
| 371 | # build/py.sh all
|
| 372 | libreadline-dev
|
| 373 | python2-dev
|
| 374 |
|
| 375 | # retrieving spec-bin -- TODO: move to build time
|
| 376 | wget
|
| 377 | # for wget https://. TODO: remove when the build is hermetic
|
| 378 | ca-certificates
|
| 379 |
|
| 380 | curl # wait for cpp-tarball
|
| 381 |
|
| 382 | gcc
|
| 383 | g++ # for C++ tarball
|
| 384 |
|
| 385 | # to build ble.sh
|
| 386 | make
|
| 387 | # used by ble.sh
|
| 388 | gawk
|
| 389 | procps
|
| 390 |
|
| 391 | # for ble.sh contra
|
| 392 | libx11-dev
|
| 393 | libxft-dev
|
| 394 | libncursesw5-dev
|
| 395 | )
|
| 396 |
|
| 397 | apt-install "${packages[@]}"
|
| 398 | }
|
| 399 |
|
| 400 | if test $(basename $0) = 'from-apt.sh'; then
|
| 401 | "$@"
|
| 402 | fi
|