OILS / mycpp / NINJA_subgraph.py View on Github | oils.pub

457 lines, 295 significant
1"""
2mycpp/NINJA_subgraph.py
3"""
4from __future__ import print_function
5
6import os
7import sys
8
9from build.ninja_lib import (log, mycpp_library, mycpp_binary,
10 COMPILERS_VARIANTS, OTHER_VARIANTS)
11
12_ = log
13
14
15def DefineTargets(ru):
16
17 # Creates _bin/shwrap/mycpp_main
18 ru.py_binary('mycpp/mycpp_main.py',
19 deps_base_dir='prebuilt/ninja',
20 template='mycpp')
21
22 # mycpp wrapper that depends on _bin/datalog/dataflow, a binary created
23 # from Souffle datalog!
24 ru.n.build(
25 '_bin/shwrap/mycpp_main_souffle',
26 'cp',
27 ['bin/mycpp_main_souffle.sh'],
28 implicit=['_bin/shwrap/mycpp_main', '_bin/datalog/dataflow'],
29 )
30
31 ru.n.build(
32 '_bin/shwrap/mycpp_main_nosouffle',
33 'cp',
34 ['bin/mycpp_main_nosouffle.sh'],
35 implicit=['_bin/shwrap/mycpp_main'],
36 )
37
38 ru.cc_library(
39 '//mycpp/runtime',
40 # Could separate into //mycpp/runtime_{marksweep,bumpleak}
41 srcs=[
42 'mycpp/bump_leak_heap.cc',
43 'mycpp/gc_iolib.cc',
44 'mycpp/gc_mylib.cc',
45 ],
46 deps=['//mycpp/runtime_pure'],
47 )
48
49 ru.cc_library('//mycpp/runtime_pure',
50 srcs=[
51 'mycpp/gc_builtins.cc',
52 'mycpp/gc_mops.cc',
53 'mycpp/gc_str.cc',
54 'mycpp/hash.cc',
55 'mycpp/mark_sweep_heap.cc',
56 ])
57
58 # Special test with -D
59 ru.cc_binary('mycpp/bump_leak_heap_test.cc',
60 deps=['//mycpp/runtime'],
61 matrix=[
62 ('cxx', 'asan+bumpleak'),
63 ('cxx', 'ubsan+bumpleak'),
64 ('clang', 'ubsan+bumpleak'),
65 ('clang', 'coverage+bumpleak'),
66 ],
67 phony_prefix='mycpp-unit')
68
69 for test_main in [
70 'mycpp/mark_sweep_heap_test.cc',
71 'mycpp/gc_heap_test.cc',
72 'mycpp/gc_stress_test.cc',
73 'mycpp/gc_builtins_test.cc',
74 'mycpp/gc_iolib_test.cc',
75 'mycpp/gc_mops_test.cc',
76 'mycpp/gc_mylib_test.cc',
77 'mycpp/gc_dict_test.cc',
78 'mycpp/gc_list_test.cc',
79 'mycpp/gc_str_test.cc',
80 'mycpp/gc_tuple_test.cc',
81 'mycpp/small_str_test.cc',
82 ]:
83 ru.cc_binary(test_main,
84 deps=['//mycpp/runtime'],
85 matrix=(COMPILERS_VARIANTS + OTHER_VARIANTS),
86 phony_prefix='mycpp-unit')
87
88 ru.cc_binary(
89 'mycpp/float_test.cc',
90 deps=['//mycpp/runtime'],
91 # Just test two compilers, in fast mode
92 matrix=[('cxx', 'opt'), ('clang', 'opt')],
93 phony_prefix='mycpp-unit')
94
95 for test_main in [
96 'mycpp/demo/gc_header.cc',
97 'mycpp/demo/hash_table.cc',
98 'mycpp/demo/target_lang.cc',
99 ]:
100 ru.cc_binary(test_main,
101 deps=['//mycpp/runtime'],
102 matrix=COMPILERS_VARIANTS,
103 phony_prefix='mycpp-unit')
104
105 # ASDL schema that examples/parse.py depends on
106 ru.asdl_library('mycpp/examples/expr.asdl')
107
108
109#
110# mycpp/examples build config
111#
112
113# TODO:
114# - Fold this dependency into a proper shwrap wrapper
115# - Make a n.build() wrapper that takes it into account automatically
116RULES_PY = 'build/ninja-rules-py.sh'
117
118# special ones in examples.sh:
119# - parse
120# - lexer_main -- these use Oil code
121# - pgen2_demo -- uses pgen2
122
123
124def ShouldSkipBuild(name):
125 if name.startswith('invalid_'):
126 return True
127
128 if name in [
129 # these use Oils code, and don't type check or compile. Maybe give
130 # up on them? pgen2_demo might be useful later.
131 'lexer_main',
132 'pgen2_demo',
133 ]:
134 return True
135
136 return False
137
138
139def ExamplesToBuild():
140 filenames = os.listdir('mycpp/examples')
141 py = [
142 name[:-3] for name in filenames
143 if name.endswith('.py') and name != '__init__.py'
144 ]
145
146 to_test = [name for name in py if not ShouldSkipBuild(name)]
147
148 result = []
149 for ex in to_test:
150 py_main = 'mycpp/examples/%s.py' % ex
151 result.append((ex, py_main))
152
153 return result
154
155
156def ShouldSkipTest(name):
157 return False
158
159
160def ShouldSkipBenchmark(name):
161 return name.startswith('test_')
162
163
164TRANSLATE_FILES = {
165 # TODO: Use build/dynamic_deps.py here
166 # BUG: modules.py must be listed last. Order matters with inheritance
167 # across modules!
168 'modules': [
169 'mycpp/testpkg/module1.py',
170 'mycpp/testpkg/module2.py',
171 'mycpp/examples/modules.py',
172 ],
173 'parse': [], # added dynamically from mycpp/examples/parse.translate.txt
174}
175
176# Unused. Could use _build/NINJA/mycpp.examples.parse/typecheck.txt
177EXAMPLES_PY = {
178 'parse': [],
179}
180
181EXAMPLES_DEPS = {
182 'parse': [
183 '//mycpp/runtime',
184 '//mycpp/examples/expr.asdl',
185 '//cpp/data_lang',
186 ],
187}
188
189# mycpp-souffle only has three variants for now
190SOUFFLE_MATRIX = [
191 ('cxx', 'opt'), # for benchmarks
192 ('cxx', 'asan'), # need this for running the examples in CI
193 ('cxx', 'asan+gcalways'),
194]
195MYPY_PATH = '$NINJA_REPO_ROOT/mycpp:$NINJA_REPO_ROOT/pyext'
196
197
198def NinjaGraph(ru):
199 n = ru.n
200
201 ru.comment('Generated by %s' % __name__)
202
203 # Running build/ninja_main.py
204 this_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
205
206 n.variable('NINJA_REPO_ROOT', os.path.dirname(this_dir))
207 n.newline()
208
209 # mycpp and pea have the same interface
210 n.rule('translate-mycpp',
211 command='_bin/shwrap/mycpp_main $mypypath $preamble_path $out $in',
212 description='mycpp $mypypath $preamble_path $out $in')
213 n.newline()
214
215 n.rule('translate-mycpp-souffle',
216 command=
217 '_bin/shwrap/mycpp_main_souffle $mypypath $preamble_path $out $in',
218 description='mycpp-souffle $mypypath $preamble_path $out $in')
219 n.newline()
220
221 n.rule('translate-mycpp-nosouffle',
222 command=
223 '_bin/shwrap/mycpp_main_nosouffle $mypypath $preamble_path $out $in',
224 description='mycpp-nosouffle $mypypath $preamble_path $out $in')
225 n.newline()
226
227 n.rule('translate-pea',
228 command='_bin/shwrap/pea_main $mypypath $preamble_path $out $in',
229 description='pea $mypypath $preamble_path $out $in')
230 n.newline()
231
232 n.rule(
233 'example-task',
234 # note: $out can be MULTIPLE FILES, shell-quoted
235 command='build/ninja-rules-py.sh example-task $name $impl $bin $out',
236 description='example-task $name $impl $bin $out')
237 n.newline()
238 n.rule(
239 'typecheck',
240 command='build/ninja-rules-py.sh typecheck $main_py $out $skip_imports',
241 description='typecheck $main_py $out $skip_imports')
242 n.newline()
243 n.rule('logs-equal',
244 command='build/ninja-rules-py.sh logs-equal $out $in',
245 description='logs-equal $out $in')
246 n.newline()
247 n.rule('benchmark-table',
248 command='build/ninja-rules-py.sh benchmark-table $out $in',
249 description='benchmark-table $out $in')
250 n.newline()
251
252 # Groups of targets. Not all of these are run by default.
253 ph = {
254 'mycpp-typecheck':
255 [], # optional: for debugging only. translation does it.
256 'mycpp-strip':
257 [], # optional: strip binaries. To see how big they are.
258
259 # Compare logs for tests AND benchmarks.
260 # It's a separate task because we have multiple variants to compare, and
261 # the timing of test/benchmark tasks should NOT include comparison.
262 'mycpp-logs-equal': [],
263
264 # NOTE: _test/benchmark-table.tsv isn't included in any phony target
265
266 # Targets dynamically added:
267 #
268 # mycpp-unit-$compiler-$variant
269 # mycpp-examples-$compiler-$variant
270 'pea-translate': [],
271 'pea-compile': [],
272 # TODO: eventually we will have pea-logs-equal, and pea-benchmark-table
273 }
274 ru.AddPhony(ph)
275
276 DefineTargets(ru)
277
278 #
279 # Build and run examples/
280 #
281
282 MycppExamples(ru, ph)
283
284 #
285 # Prebuilt
286 #
287
288 ru.souffle_binary('prebuilt/datalog/call-graph.cc')
289 ru.souffle_binary('prebuilt/datalog/dataflow.cc')
290 ru.souffle_binary('prebuilt/datalog/smoke-test.cc')
291
292
293def MycppExamples(ru, ph):
294 n = ru.n
295
296 # For simplicity, this is committed to the repo. We could also have
297 # build/dev.sh minimal generate it?
298 with open('_build/NINJA/mycpp.examples.parse/translate.txt') as f:
299 for line in f:
300 path = line.strip()
301 TRANSLATE_FILES['parse'].append(path)
302
303 examples = ExamplesToBuild()
304 #examples = ['cgi', 'containers', 'fib_iter']
305
306 ## Pea Examples
307 for ex, py_main in examples:
308 # Special case: mycpp/examples/pea_* are only translated with pea.
309 # TODO: pea examples don't have the same main()
310 py_rel_path, _ = os.path.splitext(py_main)
311
312 if ex.startswith('pea_'):
313 mycpp_library(ru,
314 py_main,
315 mypy_path=MYPY_PATH,
316 translator='pea',
317 deps=['//mycpp/runtime'])
318 mycpp_binary(ru,
319 '//%s.pea' % py_rel_path,
320 matrix=[
321 ('cxx', 'asan'),
322 ('cxx', 'opt'),
323 ])
324
325 to_compare = []
326 benchmark_tasks = []
327
328 for ex, py_main in examples:
329 if ex.startswith('pea_'): # Only non-pea examples
330 continue
331 py_rel_path, _ = os.path.splitext(py_main)
332
333 ru.comment('- mycpp/examples/%s' % ex)
334
335 # TODO: make a phony target for these, since they're not strictly necessary.
336 # Translation does everything that type checking does. Type checking only
337 # is useful for debugging.
338 t = '_test/tasks/typecheck/%s.log.txt' % ex
339 main_py = 'mycpp/examples/%s.py' % ex
340
341 # expr.asdl needs to import pylib.collections_, which doesn't type check
342 skip_imports = 'T' if (ex == 'parse') else "''"
343
344 ## Type check the example
345
346 n.build(
347 [t],
348 'typecheck',
349 # TODO: Use mycpp/examples/parse.typecheck.txt
350 EXAMPLES_PY.get(ex, []) + [main_py],
351 variables=[('main_py', main_py), ('skip_imports', skip_imports)])
352 n.newline()
353 ru.phony['mycpp-typecheck'].append(t)
354
355 ## Run example as Python
356
357 for mode in ['test', 'benchmark']:
358 prefix = '_test/tasks/%s/%s.py' % (mode, ex)
359 task_out = '%s.task.txt' % prefix
360
361 if mode == 'benchmark':
362 if ShouldSkipBenchmark(ex):
363 #log('Skipping benchmark of %s', ex)
364 continue
365 benchmark_tasks.append(task_out)
366
367 elif mode == 'test':
368 if ShouldSkipTest(ex):
369 #log('Skipping test of %s', ex)
370 continue
371
372 # TODO: This should be a Python stub!
373 log_out = '%s.log' % prefix
374 n.build([task_out, log_out],
375 'example-task',
376 EXAMPLES_PY.get(ex, []) + ['mycpp/examples/%s.py' % ex],
377 variables=[('bin', main_py), ('name', ex),
378 ('impl', 'Python')])
379
380 n.newline()
381
382 ## Translate the example 2 ways, and benchmark and test it
383
384 for translator in ['mycpp', 'mycpp-souffle']:
385
386 matrix = SOUFFLE_MATRIX if translator == 'mycpp-souffle' else COMPILERS_VARIANTS
387 phony_prefix = 'mycpp-examples' if translator == 'mycpp' else None
388 py_inputs = TRANSLATE_FILES.get(ex)
389
390 deps = EXAMPLES_DEPS.get(ex, ['//mycpp/runtime'])
391 mycpp_library(ru,
392 py_main,
393 mypy_path=MYPY_PATH,
394 translator=translator,
395 py_inputs=py_inputs,
396 deps=deps)
397
398 mycpp_binary(ru,
399 '//%s.%s' % (py_rel_path, translator),
400 template='example-unix',
401 phony_prefix=phony_prefix,
402 matrix=matrix)
403
404 # minimal
405 TEST_MATRIX = [
406 ('test', 'asan'), # TODO: asan+gcalways is better!
407 ('benchmark', 'opt'),
408 ]
409
410 # Run the binary in two ways
411 for mode, variant in TEST_MATRIX:
412 task_out = '_test/tasks/%s/%s.%s.%s.task.txt' % (
413 mode, ex, translator, variant)
414
415 if mode == 'benchmark':
416 if ShouldSkipBenchmark(ex):
417 #log('Skipping benchmark of %s', ex)
418 continue
419 benchmark_tasks.append(task_out)
420
421 elif mode == 'test':
422 if ShouldSkipTest(ex):
423 #log('Skipping test of %s', ex)
424 continue
425
426 cc_log_out = '_test/tasks/%s/%s.%s.%s.log' % (
427 mode, ex, translator, variant)
428 py_log_out = '_test/tasks/%s/%s.py.log' % (mode, ex)
429
430 to_compare.append(cc_log_out)
431 to_compare.append(py_log_out)
432
433 # Only test cxx- variant
434 b_example = '_bin/cxx-%s/mycpp/examples/%s.%s' % (variant, ex,
435 translator)
436 impl = 'C++'
437 if translator == 'mycpp-souffle':
438 impl = 'C++-Souffle'
439
440 n.build([task_out, cc_log_out],
441 'example-task', [b_example],
442 variables=[('bin', b_example), ('name', ex),
443 ('impl', impl)])
444 n.newline()
445
446 # Compare the log of all examples
447 out = '_test/mycpp-compare-passing.txt'
448 n.build([out], 'logs-equal', to_compare)
449 n.newline()
450
451 # NOTE: Don't really need this
452 ru.phony['mycpp-logs-equal'].append(out)
453
454 # Timing of benchmarks
455 out = '_test/benchmark-table.tsv'
456 n.build([out], 'benchmark-table', benchmark_tasks)
457 n.newline()