OILS / build / ninja_main.py View on Github | oilshell.org

444 lines, 227 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6
7"""
8from __future__ import print_function
9
10import cStringIO
11from glob import glob
12import os
13import sys
14
15from build import ninja_lib
16from build.ninja_lib import log
17
18from asdl import NINJA_subgraph as asdl_subgraph
19from bin import NINJA_subgraph as bin_subgraph
20from core import NINJA_subgraph as core_subgraph
21from cpp import NINJA_subgraph as cpp_subgraph
22from data_lang import NINJA_subgraph as data_lang_subgraph
23from display import NINJA_subgraph as display_subgraph
24from frontend import NINJA_subgraph as frontend_subgraph
25from ysh import NINJA_subgraph as ysh_subgraph
26from osh import NINJA_subgraph as osh_subgraph
27from mycpp import NINJA_subgraph as mycpp_subgraph
28from pea import NINJA_subgraph as pea_subgraph
29from prebuilt import NINJA_subgraph as prebuilt_subgraph
30from yaks import NINJA_subgraph as yaks_subgraph
31
32from vendor import ninja_syntax
33
34# The file Ninja runs by default.
35BUILD_NINJA = 'build.ninja'
36
37
38def TarballManifest(cc_h_files):
39 names = []
40
41 # Code we know about
42 names.extend(cc_h_files)
43
44 names.extend([
45 # Text
46 'LICENSE.txt',
47 'README-native.txt',
48 'INSTALL.txt',
49 'configure',
50 'install',
51 'doc/osh.1',
52
53 # Build Scripts
54 'build/common.sh',
55 'build/native.sh',
56
57 # These 2 are used by build/ninja-rules-cpp.sh
58 'build/py2.sh',
59 'build/dev-shell.sh',
60 'build/ninja-rules-cpp.sh',
61 'mycpp/common.sh',
62
63 # Generated
64 '_build/oils.sh',
65
66 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
67 #'_gen/frontend/help_meta.h',
68 '_gen/frontend/match.re2c.h',
69 '_gen/frontend/id_kind.asdl_c.h',
70 '_gen/frontend/types.asdl_c.h',
71 ])
72
73 # For configure
74 names.extend(glob('build/detect-*.c'))
75
76 # TODO: crawl headers
77 # We can now use the headers=[] attribute
78 names.extend(glob('mycpp/*.h'))
79 names.extend(glob('cpp/*.h'))
80
81 # ONLY the headers
82 names.extend(glob('prebuilt/*/*.h'))
83
84 names.sort() # Pass them to tar sorted
85
86 # Check for dupes here
87 unique = sorted(set(names))
88 if names != unique:
89 dupes = [n for n in names if names.count(n) > 1]
90 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
91 dupes)
92
93 for name in names:
94 print(name)
95
96
97def ShellFunctions(cc_sources, f, argv0):
98 """
99 Generate a shell script that invokes the same function that build.ninja does
100 """
101 print('''\
102#!/bin/sh
103#
104# _build/oils.sh - generated by %s
105#
106# Usage:
107# _build/oils.sh COMPILER? VARIANT? SKIP_REBUILD?
108#
109# COMPILER: 'cxx' for system compiler, or 'clang' [default cxx]
110# VARIANT: 'dbg' or 'opt' [default dbg]
111# SKIP_REBUILD: if non-empty, checks if the output exists before building
112
113. build/ninja-rules-cpp.sh
114
115OILS_PARALLEL_BUILD=${OILS_PARALLEL_BUILD:-1}
116
117_compile_one() {
118 local src=$4
119
120 echo "CXX $src"
121
122 # Delegate to function in build/ninja-rules-cpp.sh
123 if test "${_do_fork:-}" = 1; then
124 compile_one "$@" & # we will wait later
125 else
126 compile_one "$@"
127 fi
128}
129
130main() {
131 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
132
133 local compiler=${1:-cxx} # default is system compiler
134 local variant=${2:-opt} # default is optimized build
135 local skip_rebuild=${3:-} # if the output exists, skip build'
136''' % (argv0),
137 file=f)
138
139 out_dir = '_bin/$compiler-$variant-sh'
140 print(' local out_dir=%s' % out_dir, file=f)
141
142 print('''\
143 local out=$out_dir/oils-for-unix
144
145 if test -n "$skip_rebuild" && test -f "$out"; then
146 echo
147 echo "$0: SKIPPING build because $out exists"
148 echo
149 return
150 fi
151
152 echo
153 echo "$0: Building oils-for-unix: $out"
154 echo "$0: PWD = $PWD"
155 echo
156''',
157 file=f)
158
159 objects = []
160
161 in_out = []
162 for src in sorted(cc_sources):
163 # e.g. _build/obj/cxx-dbg-sh/posix.o
164 prefix, _ = os.path.splitext(src)
165 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
166 in_out.append((src, obj))
167
168 bin_dir = '_bin/$compiler-$variant-sh'
169 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
170
171 all_dirs = [bin_dir] + obj_dirs
172 # Double quote
173 all_dirs = ['"%s"' % d for d in all_dirs]
174
175 print(' mkdir -p \\', file=f)
176 print(' %s' % ' \\\n '.join(all_dirs), file=f)
177 print('', file=f)
178
179 do_fork = ''
180
181 for i, (src, obj) in enumerate(in_out):
182 obj_quoted = '"%s"' % obj
183 objects.append(obj_quoted)
184
185 # Only fork one translation unit that we know to be slow
186 if 'oils_for_unix.mycpp.cc' in src:
187 # There should only be one forked translation unit
188 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
189 assert do_fork == ''
190 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
191 else:
192 do_fork = ''
193
194 if do_fork:
195 print(' # Potentially fork this translation unit with &', file=f)
196 print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork,
197 file=f)
198 print(' %s %s' % (src, obj_quoted), file=f)
199 print('', file=f)
200
201 print(' # wait for the translation unit before linking', file=f)
202 print(' echo WAIT', file=f)
203 # time -p shows any excess parallelism on 2 cores
204 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
205 # other translation units combined!
206
207 # Timing isn't POSIX
208 #print(' time -p wait', file=f)
209 print(' wait', file=f)
210 print('', file=f)
211
212 print(' echo "LINK $out"', file=f)
213 # note: can't have spaces in filenames
214 print(' link "$compiler" "$variant" "" "$out" \\', file=f)
215 # put each object on its own line, and indent by 4
216 print(' %s' % (' \\\n '.join(objects)), file=f)
217 print('', file=f)
218
219 # Strip opt binary
220 # TODO: provide a way for the user to get symbols?
221
222 print('''\
223 local out_name=oils-for-unix
224 if test "$variant" = opt; then
225 strip -o "$out.stripped" "$out"
226
227 # Symlink to unstripped binary for benchmarking
228 # out_name=$out_name.stripped
229 fi
230
231 cd $out_dir
232 for symlink in osh ysh; do
233 # like ln -v, which we can't use portably
234 echo " $symlink -> $out_name"
235 ln -s -f $out_name $symlink
236 done
237}
238
239main "$@"
240''',
241 file=f)
242
243
244def Preprocessed(n, cc_sources):
245 # See how much input we're feeding to the compiler. Test C++ template
246 # explosion, e.g. <unordered_map>
247 #
248 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
249 # metrics/source-code.sh
250
251 pre_matrix = [
252 ('cxx', 'dbg'),
253 ('cxx', 'opt'),
254 ('clang', 'dbg'),
255 ('clang', 'opt'),
256 ]
257 for compiler, variant in pre_matrix:
258 preprocessed = []
259 for src in cc_sources:
260 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
261 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
262 preprocessed.append(pre)
263
264 # Summary file
265 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
266 'line_count', preprocessed)
267 n.newline()
268
269
270def InitSteps(n):
271 """Wrappers for build/ninja-rules-*.sh
272
273 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
274 """
275 #
276 # Compiling and linking
277 #
278
279 # Preprocess one translation unit
280 n.rule(
281 'preprocess',
282 # compile_one detects the _build/preprocessed path
283 command=
284 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
285 description='PP $compiler $variant $more_cxx_flags $in $out')
286 n.newline()
287
288 n.rule('line_count',
289 command='build/ninja-rules-cpp.sh line_count $out $in',
290 description='line_count $out $in')
291 n.newline()
292
293 # Compile one translation unit
294 n.rule(
295 'compile_one',
296 command=
297 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
298 depfile='$out.d',
299 # no prefix since the compiler is the first arg
300 description='$compiler $variant $more_cxx_flags $in $out')
301 n.newline()
302
303 # Link objects together
304 n.rule(
305 'link',
306 command=
307 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
308 description='LINK $compiler $variant $more_link_flags $out $in')
309 n.newline()
310
311 # 1 input and 2 outputs
312 n.rule('strip',
313 command='build/ninja-rules-cpp.sh strip_ $in $out',
314 description='STRIP $in $out')
315 n.newline()
316
317 # cc_binary can have symliks
318 n.rule('symlink',
319 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
320 description='SYMLINK $dir $target $new')
321 n.newline()
322
323 #
324 # Code generators
325 #
326
327 n.rule(
328 'write-shwrap',
329 # $in must start with main program
330 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
331 description='make-pystub $out $in')
332 n.newline()
333
334 n.rule(
335 'gen-oils-for-unix',
336 command=
337 'build/ninja-rules-py.sh gen-oils-for-unix $main_name $out_prefix $preamble $in',
338 description='gen-oils-for-unix $main_name $out_prefix $preamble $in')
339 n.newline()
340
341
342def main(argv):
343 try:
344 action = argv[1]
345 except IndexError:
346 action = 'ninja'
347
348 if action == 'ninja':
349 f = open(BUILD_NINJA, 'w')
350 else:
351 f = cStringIO.StringIO() # thrown away
352
353 n = ninja_syntax.Writer(f)
354 ru = ninja_lib.Rules(n)
355
356 ru.comment('InitSteps()')
357 InitSteps(n)
358
359 #
360 # Create the graph.
361 #
362
363 asdl_subgraph.NinjaGraph(ru)
364 ru.comment('')
365
366 bin_subgraph.NinjaGraph(ru)
367 ru.comment('')
368
369 core_subgraph.NinjaGraph(ru)
370 ru.comment('')
371
372 cpp_subgraph.NinjaGraph(ru)
373 ru.comment('')
374
375 data_lang_subgraph.NinjaGraph(ru)
376 ru.comment('')
377
378 display_subgraph.NinjaGraph(ru)
379 ru.comment('')
380
381 frontend_subgraph.NinjaGraph(ru)
382 ru.comment('')
383
384 mycpp_subgraph.NinjaGraph(ru)
385 ru.comment('')
386
387 ysh_subgraph.NinjaGraph(ru)
388 ru.comment('')
389
390 osh_subgraph.NinjaGraph(ru)
391 ru.comment('')
392
393 pea_subgraph.NinjaGraph(ru)
394 ru.comment('')
395
396 prebuilt_subgraph.NinjaGraph(ru)
397 ru.comment('')
398
399 yaks_subgraph.NinjaGraph(ru)
400 ru.comment('')
401
402 # Materialize all the cc_binary() rules
403 ru.WriteRules()
404
405 # Collect sources for metrics, tarball, shell script
406 cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
407
408 if 0:
409 from pprint import pprint
410 pprint(cc_sources)
411
412 # TODO: could thin these out, not generate for unit tests, etc.
413 Preprocessed(n, cc_sources)
414
415 ru.WritePhony()
416
417 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
418
419 if action == 'ninja':
420 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
421 n.num_build_targets())
422
423 elif action == 'shell':
424 out = '_build/oils.sh'
425 with open(out, 'w') as f:
426 ShellFunctions(cc_sources, f, argv[0])
427 log(' (%s) -> %s', argv[0], out)
428
429 elif action == 'tarball-manifest':
430 h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
431 TarballManifest(cc_sources + h)
432
433 else:
434 raise RuntimeError('Invalid action %r' % action)
435
436
437if __name__ == '__main__':
438 try:
439 main(sys.argv)
440 except RuntimeError as e:
441 print('FATAL: %s' % e, file=sys.stderr)
442 sys.exit(1)
443
444# vim: sw=2