OILS / mycpp / NINJA_subgraph.py View on Github | oilshell.org

419 lines, 263 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4
5from __future__ import print_function
6
7import os
8import sys
9
10from build.ninja_lib import log, COMPILERS_VARIANTS, OTHER_VARIANTS
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 ru.py_binary('mycpp/mycpp_main.py',
18 deps_base_dir='prebuilt/ninja',
19 template='mycpp')
20
21 ru.cc_library(
22 '//mycpp/runtime',
23 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
24 srcs=[
25 'mycpp/bump_leak_heap.cc',
26 'mycpp/gc_builtins.cc',
27 'mycpp/gc_mops.cc',
28 'mycpp/gc_mylib.cc',
29 'mycpp/gc_str.cc',
30 'mycpp/hash.cc',
31 'mycpp/mark_sweep_heap.cc',
32 ])
33
34 # Special test with -D
35 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
36 deps=['//mycpp/runtime'],
37 matrix=[
38 ('cxx', 'asan+bumpleak'),
39 ('cxx', 'ubsan+bumpleak'),
40 ('clang', 'ubsan+bumpleak'),
41 ('clang', 'coverage+bumpleak'),
42 ],
43 phony_prefix='mycpp-unit')
44
45 for test_main in [
46 'mycpp/mark_sweep_heap_test.cc',
47 'mycpp/gc_heap_test.cc',
48 'mycpp/gc_stress_test.cc',
49 'mycpp/gc_builtins_test.cc',
50 'mycpp/gc_mops_test.cc',
51 'mycpp/gc_mylib_test.cc',
52 'mycpp/gc_dict_test.cc',
53 'mycpp/gc_list_test.cc',
54 'mycpp/gc_str_test.cc',
55 'mycpp/gc_tuple_test.cc',
56 'mycpp/small_str_test.cc',
57 ]:
58 ru.cc_binary(test_main,
59 deps=['//mycpp/runtime'],
60 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
61 phony_prefix='mycpp-unit')
62
63 ru.cc_binary(
64 'mycpp/float_test.cc',
65 deps=['//mycpp/runtime'],
66 # Just test two compilers, in fast mode
67 matrix=[('cxx', 'opt'), ('clang', 'opt')],
68 phony_prefix='mycpp-unit')
69
70 for test_main in [
71 'mycpp/demo/gc_header.cc',
72 'mycpp/demo/hash_table.cc',
73 'mycpp/demo/target_lang.cc',
74 ]:
75 ru.cc_binary(test_main,
76 deps=['//mycpp/runtime'],
77 matrix=COMPILERS_VARIANTS,
78 phony_prefix='mycpp-unit')
79
80 # ASDL schema that examples/parse.py depends on
81 ru.asdl_library('mycpp/examples/expr.asdl')
82
83
84#
85# mycpp/examples build config
86#
87
88# TODO:
89# - Fold this dependency into a proper shwrap wrapper
90# - Make a n.build() wrapper that takes it into account automatically
91RULES_PY = 'build/ninja-rules-py.sh'
92
93# special ones in examples.sh:
94# - parse
95# - lexer_main -- these use Oil code
96# - pgen2_demo -- uses pgen2
97
98
99def ShouldSkipBuild(name):
100 if name.startswith('invalid_'):
101 return True
102
103 if name in [
104 # these use Oil code, and don't type check or compile. Maybe give up on
105 # them? pgen2_demo might be useful later.
106 'lexer_main',
107 'pgen2_demo',
108 ]:
109 return True
110
111 return False
112
113
114def ExamplesToBuild():
115 filenames = os.listdir('mycpp/examples')
116 py = [
117 name[:-3] for name in filenames
118 if name.endswith('.py') and name != '__init__.py'
119 ]
120
121 to_test = [name for name in py if not ShouldSkipBuild(name)]
122
123 return to_test
124
125
126def ShouldSkipTest(name):
127 return False
128
129
130def ShouldSkipBenchmark(name):
131 return name.startswith('test_')
132
133
134TRANSLATE_FILES = {
135 # TODO: We could also use app_deps.py here
136 # BUG: modules.py must be listed last. Order matters with inheritance
137 # across modules!
138 'modules': [
139 'mycpp/testpkg/module1.py',
140 'mycpp/testpkg/module2.py',
141 'mycpp/examples/modules.py',
142 ],
143 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
144}
145
146# Unused. Could use mycpp/examples/parse.typecheck.txt
147EXAMPLES_PY = {
148 'parse': [],
149}
150
151
152def TranslatorSubgraph(ru, translator, ex):
153 n = ru.n
154
155 raw = '_gen/mycpp/examples/%s_raw.%s.cc' % (ex, translator)
156
157 # Translate to C++
158 if ex in TRANSLATE_FILES:
159 to_translate = TRANSLATE_FILES[ex]
160 else:
161 to_translate = ['mycpp/examples/%s.py' % ex]
162
163 # Implicit dependency: if the translator changes, regenerate source code.
164 # But don't pass it on the command line.
165 translator_wrapper = '_bin/shwrap/%s_main' % translator
166
167 n.build(
168 raw,
169 'translate-%s' % translator,
170 to_translate,
171 implicit=[translator_wrapper],
172 # examples/parse uses pyext/fastfunc.pyi
173 variables=[('mypypath',
174 '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext')])
175
176 p = 'mycpp/examples/%s_preamble.h' % ex
177 # Ninja empty string!
178 preamble_path = p if os.path.exists(p) else "''"
179
180 main_cc_src = '_gen/mycpp/examples/%s.%s.cc' % (ex, translator)
181
182 # Make a translation unit
183 n.build(main_cc_src,
184 'wrap-cc',
185 raw,
186 implicit=[RULES_PY],
187 variables=[('name', ex), ('preamble_path', preamble_path),
188 ('translator', translator)])
189
190 n.newline()
191
192 if translator == 'pea':
193 ru.phony['pea-translate'].append(main_cc_src)
194
195 if translator == 'mycpp':
196 example_matrix = COMPILERS_VARIANTS
197 else:
198 # pea just has one variant for now
199 example_matrix = [('cxx', 'asan+gcalways')]
200
201 if translator == 'mycpp':
202 phony_prefix = 'mycpp-examples'
203 else:
204 phony_prefix = ''
205
206 deps = ['//mycpp/runtime']
207 if ex == 'parse':
208 deps = deps + ['//mycpp/examples/expr.asdl', '//cpp/data_lang']
209
210 ru.cc_binary(
211 main_cc_src,
212 deps=deps,
213 matrix=example_matrix,
214 phony_prefix=phony_prefix,
215 )
216
217 # TODO:
218 # - restore lost 'pea-compile' tag?
219
220
221def NinjaGraph(ru):
222 n = ru.n
223
224 ru.comment('Generated by %s' % __name__)
225
226 # Running build/ninja_main.py
227 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
228
229 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
230 n.newline()
231
232 # mycpp and pea have the same interface
233 n.rule('translate-mycpp',
234 command='_bin/shwrap/mycpp_main $mypypath $out $in',
235 description='mycpp $mypypath $out $in')
236 n.newline()
237
238 n.rule('translate-pea',
239 command='_bin/shwrap/pea_main $mypypath $out $in',
240 description='pea $mypypath $out $in')
241 n.newline()
242
243 n.rule(
244 'wrap-cc',
245 command=
246 'build/ninja-rules-py.sh wrap-cc $out $translator $name $in $preamble_path',
247 description='wrap-cc $out $translator $name $in $preamble_path $out')
248 n.newline()
249 n.rule(
250 'example-task',
251 # note: $out can be MULTIPLE FILES, shell-quoted
252 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
253 description='example-task $name $impl $bin $out')
254 n.newline()
255 n.rule(
256 'typecheck',
257 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
258 description='typecheck $main_py $out $skip_imports')
259 n.newline()
260 n.rule('logs-equal',
261 command='build/ninja-rules-py.sh logs-equal $out $in',
262 description='logs-equal $out $in')
263 n.newline()
264 n.rule('benchmark-table',
265 command='build/ninja-rules-py.sh benchmark-table $out $in',
266 description='benchmark-table $out $in')
267 n.newline()
268
269 # For simplicity, this is committed to the repo. We could also have
270 # build/dev.sh minimal generate it?
271 with open('mycpp/examples/parse.translate.txt') as f:
272 for line in f:
273 path = line.strip()
274 TRANSLATE_FILES['parse'].append(path)
275
276 examples = ExamplesToBuild()
277 #examples = ['cgi', 'containers', 'fib_iter']
278
279 # Groups of targets. Not all of these are run by default.
280 ph = {
281 'mycpp-typecheck':
282 [], # optional: for debugging only. translation does it.
283 'mycpp-strip':
284 [], # optional: strip binaries. To see how big they are.
285
286 # Compare logs for tests AND benchmarks.
287 # It's a separate task because we have multiple variants to compare, and
288 # the timing of test/benchmark tasks should NOT include comparison.
289 'mycpp-logs-equal': [],
290
291 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
292
293 # Targets dynamically added:
294 #
295 # mycpp-unit-$compiler-$variant
296 # mycpp-examples-$compiler-$variant
297 'pea-translate': [],
298 'pea-compile': [],
299 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
300 }
301 ru.AddPhony(ph)
302
303 DefineTargets(ru)
304
305 #
306 # Build and run examples/
307 #
308
309 to_compare = []
310 benchmark_tasks = []
311
312 for ex in examples:
313 ru.comment('- mycpp/examples/%s' % ex)
314
315 # TODO: make a phony target for these, since they're not strictly necessary.
316 # Translation does everything that type checking does. Type checking only
317 # is useful for debugging.
318 t = '_test/tasks/typecheck/%s.log.txt' % ex
319 main_py = 'mycpp/examples/%s.py' % ex
320
321 # expr.asdl needs to import pylib.collections_, which doesn't type check
322 skip_imports = 'T' if (ex == 'parse') else "''"
323
324 n.build(
325 [t],
326 'typecheck',
327 # TODO: Use mycpp/examples/parse.typecheck.txt
328 EXAMPLES_PY.get(ex, []) + [main_py],
329 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
330 n.newline()
331 ru.phony['mycpp-typecheck'].append(t)
332
333 # Run Python.
334 for mode in ['test', 'benchmark']:
335 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
336 task_out = '%s.task.txt' % prefix
337
338 if mode == 'benchmark':
339 if ShouldSkipBenchmark(ex):
340 #log('Skipping benchmark of %s', ex)
341 continue
342 benchmark_tasks.append(task_out)
343
344 elif mode == 'test':
345 if ShouldSkipTest(ex):
346 #log('Skipping test of %s', ex)
347 continue
348
349 # TODO: This should be a Python stub!
350 log_out = '%s.log' % prefix
351 n.build([task_out, log_out],
352 'example-task',
353 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
354 variables=[('bin', main_py), ('name', ex),
355 ('impl', 'Python')])
356
357 n.newline()
358
359 for translator in ['mycpp', 'pea']:
360 TranslatorSubgraph(ru, translator, ex)
361
362 # Don't run it for now; just compile
363 if translator == 'pea':
364 continue
365
366 # minimal
367 MATRIX = [
368 ('test', 'asan'), # TODO: asan+gcalways is better!
369 ('benchmark', 'opt'),
370 ]
371
372 # Run the binary in two ways
373 for mode, variant in MATRIX:
374 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
375 mode, ex, translator, variant)
376
377 if mode == 'benchmark':
378 if ShouldSkipBenchmark(ex):
379 #log('Skipping benchmark of %s', ex)
380 continue
381 benchmark_tasks.append(task_out)
382
383 elif mode == 'test':
384 if ShouldSkipTest(ex):
385 #log('Skipping test of %s', ex)
386 continue
387
388 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
389 mode, ex, translator, variant)
390 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
391
392 to_compare.append(cc_log_out)
393 to_compare.append(py_log_out)
394
395 # Only test cxx- variant
396 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
397 translator)
398 n.build([task_out, cc_log_out],
399 'example-task', [b_example],
400 variables=[('bin', b_example), ('name', ex),
401 ('impl', 'C++')])
402 n.newline()
403
404 # Compare the log of all examples
405 out = '_test/mycpp-compare-passing.txt'
406 n.build([out], 'logs-equal', to_compare)
407 n.newline()
408
409 # NOTE: Don't really need this
410 ru.phony['mycpp-logs-equal'].append(out)
411
412 # Timing of benchmarks
413 out = '_test/benchmark-table.tsv'
414 n.build([out], 'benchmark-table', benchmark_tasks)
415 n.newline()
416
417 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
418 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
419 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')