OILS / build / ninja_main.py View on Github | oilshell.org

429 lines, 210 significant
1#!/usr/bin/env python2
2"""
3build/ninja_main.py - invoked by ./NINJA-config.sh
4
5See build/README.md for the code and data layout.
6
7"""
8from __future__ import print_function
9
10import cStringIO
11from glob import glob
12import os
13import sys
14
15from build import ninja_lib
16from build.ninja_lib import log
17
18from asdl import NINJA_subgraph as asdl_subgraph
19from bin import NINJA_subgraph as bin_subgraph
20from core import NINJA_subgraph as core_subgraph
21from cpp import NINJA_subgraph as cpp_subgraph
22from data_lang import NINJA_subgraph as data_lang_subgraph
23from frontend import NINJA_subgraph as frontend_subgraph
24from ysh import NINJA_subgraph as ysh_subgraph
25from osh import NINJA_subgraph as osh_subgraph
26from mycpp import NINJA_subgraph as mycpp_subgraph
27from pea import NINJA_subgraph as pea_subgraph
28from prebuilt import NINJA_subgraph as prebuilt_subgraph
29from yaks import NINJA_subgraph as yaks_subgraph
30
31from vendor import ninja_syntax
32
33
34# The file Ninja runs by default.
35BUILD_NINJA = 'build.ninja'
36
37
38def TarballManifest(cc_h_files):
39 names = []
40
41 # Code we know about
42 names.extend(cc_h_files)
43
44 names.extend([
45 # Text
46 'LICENSE.txt',
47 'README-native.txt',
48 'configure',
49 'install',
50 'doc/osh.1',
51
52 # Build Scripts
53 'build/common.sh',
54 'build/native.sh',
55
56 # These 2 are used by build/ninja-rules-cpp.sh
57 'build/py2.sh',
58 'build/dev-shell.sh',
59
60 'build/ninja-rules-cpp.sh',
61 'mycpp/common.sh',
62
63 # Generated
64 '_build/oils.sh',
65
66 # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
67 #'_gen/frontend/help_meta.h',
68 '_gen/frontend/match.re2c.h',
69 '_gen/frontend/id_kind.asdl_c.h',
70 '_gen/frontend/types.asdl_c.h',
71 ])
72
73 # For configure
74 names.extend(glob('build/detect-*.c'))
75
76 # TODO: crawl headers
77 # We can now use the headers=[] attribute
78 names.extend(glob('mycpp/*.h'))
79 names.extend(glob('cpp/*.h'))
80
81 # ONLY the headers
82 names.extend(glob('prebuilt/*/*.h'))
83
84 names.sort() # Pass them to tar sorted
85
86 # Check for dupes here
87 unique = sorted(set(names))
88 if names != unique:
89 dupes = [n for n in names if names.count(n) > 1]
90 raise AssertionError("Tarball manifest shouldn't have duplicates: %s" % dupes)
91
92 for name in names:
93 print(name)
94
95
96def ShellFunctions(cc_sources, f, argv0):
97 """
98 Generate a shell script that invokes the same function that build.ninja does
99 """
100 print('''\
101#!/bin/sh
102#
103# _build/oils.sh - generated by %s
104#
105# Usage:
106# _build/oils.sh COMPILER? VARIANT? SKIP_REBUILD?
107#
108# COMPILER: 'cxx' for system compiler, or 'clang' [default cxx]
109# VARIANT: 'dbg' or 'opt' [default dbg]
110# SKIP_REBUILD: if non-empty, checks if the output exists before building
111
112. build/ninja-rules-cpp.sh
113
114OILS_PARALLEL_BUILD=${OILS_PARALLEL_BUILD:-1}
115
116_compile_one() {
117 local src=$4
118
119 echo "CXX $src"
120
121 # Delegate to function in build/ninja-rules-cpp.sh
122 if test "${_do_fork:-}" = 1; then
123 compile_one "$@" & # we will wait later
124 else
125 compile_one "$@"
126 fi
127}
128
129main() {
130 ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
131
132 local compiler=${1:-cxx} # default is system compiler
133 local variant=${2:-opt} # default is optimized build
134 local skip_rebuild=${3:-} # if the output exists, skip build'
135''' % (argv0), file=f)
136
137 out_dir = '_bin/$compiler-$variant-sh'
138 print(' local out_dir=%s' % out_dir, file=f)
139
140 print('''\
141 local out=$out_dir/oils-for-unix
142
143 if test -n "$skip_rebuild" && test -f "$out"; then
144 echo
145 echo "$0: SKIPPING build because $out exists"
146 echo
147 return
148 fi
149
150 echo
151 echo "$0: Building oils-for-unix: $out"
152 echo
153''', file=f)
154
155 objects = []
156
157 in_out = []
158 for src in sorted(cc_sources):
159 # e.g. _build/obj/cxx-dbg-sh/posix.o
160 prefix, _ = os.path.splitext(src)
161 obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
162 in_out.append((src, obj))
163
164 bin_dir = '_bin/$compiler-$variant-sh'
165 obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
166
167 all_dirs = [bin_dir] + obj_dirs
168 # Double quote
169 all_dirs = ['"%s"' % d for d in all_dirs]
170
171 print(' mkdir -p \\', file=f)
172 print(' %s' % ' \\\n '.join(all_dirs), file=f)
173 print('', file=f)
174
175 do_fork = ''
176
177 for i, (src, obj) in enumerate(in_out):
178 obj_quoted = '"%s"' % obj
179 objects.append(obj_quoted)
180
181 # Only fork one translation unit that we know to be slow
182 if 'oils_for_unix.mycpp.cc' in src:
183 # There should only be one forked translation unit
184 # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
185 assert do_fork == ''
186 do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
187 else:
188 do_fork = ''
189
190 if do_fork:
191 print(' # Potentially fork this translation unit with &', file=f)
192 print(' %s _compile_one "$compiler" "$variant" "" \\' % do_fork, file=f)
193 print(' %s %s' % (src, obj_quoted), file=f)
194 print('', file=f)
195
196 print(' # wait for the translation unit before linking', file=f)
197 print(' echo WAIT', file=f)
198 # time -p shows any excess parallelism on 2 cores
199 # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
200 # other translation units combined!
201
202 # Timing isn't POSIX
203 #print(' time -p wait', file=f)
204 print(' wait', file=f)
205 print('', file=f)
206
207 print(' echo "LINK $out"', file=f)
208 # note: can't have spaces in filenames
209 print(' link "$compiler" "$variant" "$out" \\', file=f)
210 # put each object on its own line, and indent by 4
211 print(' %s' % (' \\\n '.join(objects)), file=f)
212 print('', file=f)
213
214 # Strip opt binary
215 # TODO: provide a way for the user to get symbols?
216
217 print('''\
218 local out_name=oils-for-unix
219 if test "$variant" = opt; then
220 strip -o "$out.stripped" "$out"
221
222 # Symlink to unstripped binary for benchmarking
223 # out_name=$out_name.stripped
224 fi
225
226 cd $out_dir
227 for symlink in osh ysh; do
228 # like ln -v, which we can't use portably
229 echo " $symlink -> $out_name"
230 ln -s -f $out_name $symlink
231 done
232}
233
234main "$@"
235''', file=f)
236
237
238def Preprocessed(n, cc_sources):
239 # See how much input we're feeding to the compiler. Test C++ template
240 # explosion, e.g. <unordered_map>
241 #
242 # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
243 # metrics/source-code.sh
244
245 pre_matrix = [
246 ('cxx', 'dbg'),
247 ('cxx', 'opt'),
248 ('clang', 'dbg'),
249 ('clang', 'opt'),
250 ]
251 for compiler, variant in pre_matrix:
252 preprocessed = []
253 for src in cc_sources:
254 # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
255 pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
256 preprocessed.append(pre)
257
258 # Summary file
259 n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
260 'line_count',
261 preprocessed)
262 n.newline()
263
264
265def InitSteps(n):
266 """Wrappers for build/ninja-rules-*.sh
267
268 Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
269 """
270
271 #
272 # Compiling and linking
273 #
274
275 # Preprocess one translation unit
276 n.rule('preprocess',
277 # compile_one detects the _build/preprocessed path
278 command='build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
279 description='PP $compiler $variant $more_cxx_flags $in $out')
280 n.newline()
281
282 n.rule('line_count',
283 command='build/ninja-rules-cpp.sh line_count $out $in',
284 description='line_count $out $in')
285 n.newline()
286
287 # Compile one translation unit
288 n.rule('compile_one',
289 command='build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
290 depfile='$out.d',
291 # no prefix since the compiler is the first arg
292 description='$compiler $variant $more_cxx_flags $in $out')
293 n.newline()
294
295 # Link objects together
296 n.rule('link',
297 command='build/ninja-rules-cpp.sh link $compiler $variant $out $in',
298 description='LINK $compiler $variant $out $in')
299 n.newline()
300
301 # 1 input and 2 outputs
302 n.rule('strip',
303 command='build/ninja-rules-cpp.sh strip_ $in $out',
304 description='STRIP $in $out')
305 n.newline()
306
307 # cc_binary can have symliks
308 n.rule('symlink',
309 command='build/ninja-rules-cpp.sh symlink $dir $target $new',
310 description='SYMLINK $dir $target $new')
311 n.newline()
312
313 #
314 # Code generators
315 #
316
317 n.rule('write-shwrap',
318 # $in must start with main program
319 command='build/ninja-rules-py.sh write-shwrap $template $out $in',
320 description='make-pystub $out $in')
321 n.newline()
322
323 n.rule('gen-oils-for-unix',
324 command='build/ninja-rules-py.sh gen-oils-for-unix $main_name $out_prefix $preamble $in',
325 description='gen-oils-for-unix $main_name $out_prefix $preamble $in')
326 n.newline()
327
328
329def main(argv):
330 try:
331 action = argv[1]
332 except IndexError:
333 action = 'ninja'
334
335 if action == 'ninja':
336 f = open(BUILD_NINJA, 'w')
337 else:
338 f = cStringIO.StringIO() # thrown away
339
340 n = ninja_syntax.Writer(f)
341 ru = ninja_lib.Rules(n)
342
343 ru.comment('InitSteps()')
344 InitSteps(n)
345
346 #
347 # Create the graph.
348 #
349
350 asdl_subgraph.NinjaGraph(ru)
351 ru.comment('')
352
353 bin_subgraph.NinjaGraph(ru)
354 ru.comment('')
355
356 core_subgraph.NinjaGraph(ru)
357 ru.comment('')
358
359 cpp_subgraph.NinjaGraph(ru)
360 ru.comment('')
361
362 data_lang_subgraph.NinjaGraph(ru)
363 ru.comment('')
364
365 frontend_subgraph.NinjaGraph(ru)
366 ru.comment('')
367
368 mycpp_subgraph.NinjaGraph(ru)
369 ru.comment('')
370
371 ysh_subgraph.NinjaGraph(ru)
372 ru.comment('')
373
374 osh_subgraph.NinjaGraph(ru)
375 ru.comment('')
376
377 pea_subgraph.NinjaGraph(ru)
378 ru.comment('')
379
380 prebuilt_subgraph.NinjaGraph(ru)
381 ru.comment('')
382
383 yaks_subgraph.NinjaGraph(ru)
384 ru.comment('')
385
386
387 # Materialize all the cc_binary() rules
388 ru.WriteRules()
389
390 # Collect sources for metrics, tarball, shell script
391 cc_sources = ru.SourcesForBinary('_gen/bin/oils_for_unix.mycpp.cc')
392
393 if 0:
394 from pprint import pprint
395 pprint(cc_sources)
396
397 # TODO: could thin these out, not generate for unit tests, etc.
398 Preprocessed(n, cc_sources)
399
400 ru.WritePhony()
401
402 n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
403
404 if action == 'ninja':
405 log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
406 n.num_build_targets())
407
408 elif action == 'shell':
409 out = '_build/oils.sh'
410 with open(out, 'w') as f:
411 ShellFunctions(cc_sources, f, argv[0])
412 log(' (%s) -> %s', argv[0], out)
413
414 elif action == 'tarball-manifest':
415 h = ru.HeadersForBinary('_gen/bin/oils_for_unix.mycpp.cc')
416 TarballManifest(cc_sources + h)
417
418 else:
419 raise RuntimeError('Invalid action %r' % action)
420
421
422if __name__ == '__main__':
423 try:
424 main(sys.argv)
425 except RuntimeError as e:
426 print('FATAL: %s' % e, file=sys.stderr)
427 sys.exit(1)
428
429# vim: sw=2