OILS / build / ovm-compile.sh View on Github | oilshell.org

430 lines, 168 significant
1#!/usr/bin/env bash
2#
3# Compile OVM tarball.
4#
5# Usage:
6# build/ovm-compile.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11shopt -s strict:all 2>/dev/null || true # dogfood for OSH
12
13REPO_ROOT=$(cd $(dirname $0)/..; pwd)
14readonly REPO_ROOT
15
16source build/common.sh
17
18source-detected-config-or-die() {
19 if ! source _build/detected-config.sh; then
20 # Make this error stand out.
21 echo
22 echo "FATAL: can't find _build/detected-config.h. Run './configure'"
23 echo
24 exit 1
25 fi
26}
27
28# NOTES on trying to delete certain modules:
29#
30# _warnings.c: There weren't that many; it probably could be deleted.
31# bufferobject.c: the types.py module uses it.
32# Python-ast.h: pythonrun.c uses it in several places (mod_ty), and a lot of
33# stuff uses pythonrun.c.
34# pythonrun.c: lots interpreter flags and interpreter initialization caused
35# link errors.
36# pyctype.c: Tables needed for many string operations.
37
38# getargs.c: needed for Python-C API, e.g. PyArg_ParseTuple.
39# dtoa.c: not tried, but I assume that %.3f for 'time' uses it.
40
41
42readonly OVM_PYTHON_OBJS='
43Python/_warnings.c
44Python/bltinmodule.c
45Python/ceval.c
46Python/errors.c
47Python/getargs.c
48Python/getcompiler.c
49Python/getplatform.c
50Python/getversion.c
51Python/import.c
52Python/marshal.c
53Python/modsupport.c
54Python/mystrtoul.c
55Python/mysnprintf.c
56Python/pyarena.c
57Python/pyctype.c
58Python/pyfpe.c
59Python/pystate.c
60Python/pythonrun.c
61Python/random.c
62Python/structmember.c
63Python/sysmodule.c
64Python/traceback.c
65Python/pystrtod.c
66Python/dtoa.c
67Python/pymath.c
68'
69# NOTE: pystrtod.c needs some floating point functions in pymath.c
70
71OBJECT_OBJS='
72Objects/abstract.c
73Objects/boolobject.c
74Objects/bufferobject.c
75Objects/bytes_methods.c
76Objects/capsule.c
77Objects/cellobject.c
78Objects/classobject.c
79Objects/cobject.c
80Objects/codeobject.c
81Objects/descrobject.c
82Objects/enumobject.c
83Objects/exceptions.c
84Objects/genobject.c
85Objects/fileobject.c
86Objects/floatobject.c
87Objects/frameobject.c
88Objects/funcobject.c
89Objects/intobject.c
90Objects/iterobject.c
91Objects/listobject.c
92Objects/longobject.c
93Objects/dictobject.c
94Objects/methodobject.c
95Objects/moduleobject.c
96Objects/object.c
97Objects/obmalloc.c
98Objects/rangeobject.c
99Objects/setobject.c
100Objects/sliceobject.c
101Objects/stringobject.c
102Objects/structseq.c
103Objects/tupleobject.c
104Objects/typeobject.c
105Objects/weakrefobject.c
106'
107
108# Non-standard lib stuff.
109MODULE_OBJS='
110Modules/main.c
111Modules/gcmodule.c
112'
113
114# The stuff in Modules/Setup.dist, signalmodule.c. NOTE: In Python,
115# signalmodule.c is specified in Modules/Setup.config, which comes from
116# 'configure' output.
117MODOBJS='
118Modules/errnomodule.c
119Modules/pwdmodule.c
120Modules/_weakref.c
121Modules/zipimport.c
122Modules/signalmodule.c
123'
124
125# Parser/myreadline.c is needed for raw_input() to work. There is a dependency
126# from Python/bltinmodule.c to it.
127OVM_LIBRARY_OBJS="
128Modules/getbuildinfo.c
129Parser/myreadline.c
130$OBJECT_OBJS
131$OVM_PYTHON_OBJS
132$MODULE_OBJS
133$MODOBJS
134"
135
136readonly EMPTY_STR='""'
137
138# Stub out a few variables
139readonly PREPROC_FLAGS=(
140 -D OVM_MAIN \
141 -D PYTHONPATH="$EMPTY_STR" \
142 -D VERSION="$EMPTY_STR" \
143 -D VPATH="$EMPTY_STR" \
144 -D Py_BUILD_CORE \
145 # Python already has support for disabling complex numbers!
146 -D WITHOUT_COMPLEX
147)
148
149# NOTE: build/oil-defs is hard-coded to the oil.ovm app. We're abandoning
150# hello.ovm and opy.ovm for now, but those can easily be added later. We
151# haven't mangled the CPython source!
152readonly INCLUDE_PATHS=(
153 -I . # for pyconfig.h
154 -I .. # for _gen/frontend/id_kind_asdl_c.h etc.
155 -I Include
156 -I ../build/oil-defs
157)
158readonly CC=${CC:-cc} # cc should be on POSIX systems
159
160# BASE_CFLAGS is copied by observation from what configure.ac does on my Ubuntu
161# 16.04 system. Then we check if it works on Alpine Linux too.
162
163# "Python violates C99 rules, by casting between incompatible pointer types.
164# GCC may generate bad code as a result of that, so use -fno-strict-aliasing if
165# supported."
166# - gcc 4.x and Clang need -fwrapv
167
168# TODO:
169# - -DNDEBUG is also passed. That turns off asserts. Do we want that?
170# - We should auto-detect the flags in configure, or simplify the source so it
171# isn't necessary. Python's configure.ac sometimes does it by compiling a test
172# file; at other times it does it by grepping $CC --help.
173
174# pyext/fanos.c needs -std=c99
175BASE_CFLAGS='-fno-strict-aliasing -fwrapv -Wall -Wstrict-prototypes -std=c99'
176
177# These flags are disabled for OS X. I would have thought it would work in
178# Clang? It works with both GCC and Clang on Linux.
179# https://stackoverflow.com/questions/6687630/how-to-remove-unused-c-c-symbols-with-gcc-and-ld
180#BASE_CFLAGS="$BASE_CFLAGS -fdata-sections -ffunction-sections"
181
182# Needed after cpython-defs filtering.
183BASE_CFLAGS="$BASE_CFLAGS -Wno-unused-variable -Wno-unused-function"
184readonly BASE_CFLAGS
185
186BASE_LDFLAGS=''
187# Disabled for OS X
188# BASE_LDFLAGS='-Wl,--gc-sections'
189
190# The user should be able to customize CFLAGS, but it shouldn't disable what's
191# in BASE_CFLAGS.
192readonly CFLAGS=${CFLAGS:-}
193readonly LDFLAGS=${LDFLAGS:-}
194
195build() {
196 local out=${1:-$PY27/ovm2}
197 local module_init=${2:-$PY27/Modules/config.c}
198 local main_name=${3:-_tmp/hello/main_name.c}
199 local c_module_srcs=${4:-_tmp/hello/c-module-srcs.txt}
200 shift 4
201
202 local abs_out=$PWD/$out
203 local abs_module_init=$PWD/$module_init
204 local abs_main_name=$PWD/$main_name
205 local abs_c_module_srcs=$PWD/$c_module_srcs
206
207 #echo $OVM_LIBRARY_OBJS
208
209 # HAVE_READLINE defined in detected-config.sh.
210 source-detected-config-or-die
211
212 pushd $PY27
213
214 local readline_flags=''
215 if [[ "$HAVE_READLINE" -eq 1 ]]; then
216 # Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c.
217 # For now, we are using raw_input() for the REPL. TODO: Parameterize this!
218 # We should create a special no_readline_raw_input().
219
220 c_module_src_list=$(cat $abs_c_module_srcs)
221
222 if [[ -n "$READLINE_DIR" ]]; then
223 readline_flags+="-L $READLINE_DIR/lib -I $READLINE_DIR/include "
224 fi
225
226 # NOTE: pyconfig.h has HAVE_LIBREADLINE but doesn't appear to use it?
227 readline_flags+="-l readline -D HAVE_READLINE"
228 else
229 # don't fail
230 c_module_src_list=$(grep -E -v '/readline.c|/line_input.c' $abs_c_module_srcs || true)
231 fi
232
233 # $PREFIX comes from ./configure and defaults to /usr/local.
234 # $EXEC_PREFIX is a GNU thing and used in getpath.c. Could probably get rid
235 # of it.
236
237 time $CC \
238 ${BASE_CFLAGS} \
239 ${CFLAGS} \
240 "${INCLUDE_PATHS[@]}" \
241 "${PREPROC_FLAGS[@]}" \
242 -D PREFIX="\"$PREFIX\"" \
243 -D EXEC_PREFIX="\"$PREFIX\"" \
244 -o $abs_out \
245 $OVM_LIBRARY_OBJS \
246 $abs_module_init \
247 $abs_main_name \
248 $c_module_src_list \
249 Modules/ovm.c \
250 -l m \
251 ${BASE_LDFLAGS} \
252 ${LDFLAGS} \
253 $readline_flags \
254 "$@"
255
256 # NOTE:
257 # -l readline -l termcap -- for Python readline. Hm it builds without -l
258 # termcap.
259 # -l z WOULD be needed for zlibmodule.c, but we don't need it because our zip
260 # file has no compression -- see build/make_zip.py with ZIP_STORED.
261 # zipimport works fine without this.
262}
263
264# build the optimized one. Makefile uses -O3.
265
266# Clang -O2 is 1.37 MB. 18 seconds to compile.
267# -m32 is 1.12 MB. But I probably have to redefine a few things because
268# there are more warnings.
269# -O3 is 1.40 MB.
270
271# GCC -O2 is 1.35 MB. 21 seconds to compile.
272
273build-dbg() {
274 build "$@" -O0 -g -D OVM_DEBUG
275}
276
277# This will be stripped later.
278build-opt() {
279 # frame pointer for perf. Otherwise stack traces are messed up!
280 # http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html#C But why
281 # isn't debuginfo enough? Because it's a recursive function?
282 # Does this make things slower? Do I need a "perf" build?
283 build "$@" -O3 -fno-omit-frame-pointer
284}
285
286#
287# Source Release (uses same files
288#
289
290add-py27() {
291 xargs -I {} -- echo $PY27/{}
292}
293
294python-sources() {
295 echo "$OVM_LIBRARY_OBJS" | add-py27
296}
297
298_headers() {
299 local c_module_srcs=${1:-_tmp/hello/c-module-srcs.txt}
300 local abs_c_module_srcs=$PWD/$c_module_srcs
301
302 cd $PY27
303
304 # -MM: no system headers
305 gcc \
306 "${INCLUDE_PATHS[@]}" \
307 "${PREPROC_FLAGS[@]}" \
308 -MM $OVM_LIBRARY_OBJS \
309 Modules/ovm.c \
310 $(cat $abs_c_module_srcs)
311}
312
313# NOTE: 91 headers in Include, but only 81 referenced here. So it's worth it.
314# These are probably for the parser.
315#
316# NOTE: We also should get rid of asdl.h and so forth.
317
318python-headers() {
319 local c_module_srcs=$1
320
321 # 1. -MM outputs Makefile fragments, so egrep turns those into proper lines.
322 #
323 # 2. The user should generated detected-config.h, so remove it.
324 #
325 # 3. # gcc outputs paths like
326 # Python-2.7.13/Python/../Objects/stringlib/stringdefs.h
327 # but 'Python/..' causes problems for tar.
328 #
329
330 # NOTE: need .def for build/oil-defs.
331 _headers $c_module_srcs \
332 | egrep --only-matching '[^ ]+\.(h|def)' \
333 | grep -v '_build/detected-config.h' \
334 | sed 's|^Python/../||' \
335 | sort | uniq | add-py27
336}
337
338make-tar() {
339 local app_name=${1:-hello}
340 local bytecode_zip=${2:-bytecode-cpython.zip}
341 local out=${3:-_release/hello.tar}
342
343 local version_file
344 case $app_name in
345 oil)
346 version_file=oil-version.txt
347 ;;
348 hello)
349 version_file=build/testdata/hello-version.txt
350 ;;
351 *)
352 die "Unknown app $app_name"
353 exit 1
354 ;;
355 esac
356 local version=$(head -n 1 $version_file)
357
358 echo "Creating $app_name version $version"
359
360 local c_module_srcs=_build/$app_name/c-module-srcs.txt
361
362 # Add oil-0.0.0/ to the beginning of every path.
363 local sed_expr="s,^,${app_name}-${version}/,"
364
365 # Differences between tarball and repo:
366 #
367 # - build/portable-rules.mk is intentionally not included in the release tarball.
368 # The Makefile can and should operate without it.
369 #
370 # - We include intermediate files like c-module-srcs.txt, so we don't have to
371 # ship tools dynamic_deps.py. The end-user build shouldn't depend on Python.
372
373 # Note: python-headers runs gcc -M, including pyconfig.h and
374 # _build/detected-config.h.
375
376 tar --create --transform "$sed_expr" --file $out \
377 LICENSE.txt \
378 INSTALL-old.txt \
379 configure \
380 install \
381 uninstall \
382 Makefile \
383 doc/osh.1 \
384 build/ovm-compile.sh \
385 build/ovm-actions.sh \
386 build/clean.sh \
387 build/common.sh \
388 build/detect-*.c \
389 _build/$app_name/$bytecode_zip \
390 _build/$app_name/*.c \
391 $PY27/LICENSE \
392 $PY27/Modules/ovm.c \
393 $c_module_srcs \
394 $(cat $c_module_srcs | add-py27) \
395 $(python-headers $c_module_srcs) \
396 $(python-sources)
397
398 ls -l $out
399}
400
401# 123K lines.
402# Excluding MODOBJS, it's 104K lines.
403#
404# Biggest: posixmodule,unicodeobject,typeobject,ceval.
405#
406# Remove tmpnam from posixmodule, other cruft.
407#
408# Big ones to rid of: unicodeobject.c, import.c
409# codecs and codecsmodule? There is some non-unicode stuff there though.
410#
411# Probably need unicode for compatibility with modules and web frameworks
412# especially.
413
414count-c-lines() {
415 pushd $PY27
416 wc -l $OVM_LIBRARY_OBJS | sort -n
417
418 # 90 files.
419 # NOTE: To count headers, use the tar file.
420 echo
421 echo 'Files:'
422 { for i in $OVM_LIBRARY_OBJS; do
423 echo $i
424 done
425 } | wc -l
426
427 popd
428}
429
430"$@"