| 1 | #!/usr/bin/env python2
|
| 2 | """
|
| 3 | build/ninja_main.py - invoked by ./NINJA-config.sh
|
| 4 |
|
| 5 | See build/README.md for the code and data layout.
|
| 6 | """
|
| 7 | from __future__ import print_function
|
| 8 |
|
| 9 | import cStringIO
|
| 10 | from glob import glob
|
| 11 | import os
|
| 12 | import re
|
| 13 | import sys
|
| 14 |
|
| 15 | from build import ninja_lib
|
| 16 | from build.ninja_lib import log
|
| 17 |
|
| 18 | from asdl import NINJA_subgraph as asdl_subgraph
|
| 19 | from bin import NINJA_subgraph as bin_subgraph
|
| 20 | from core import NINJA_subgraph as core_subgraph
|
| 21 | from cpp import NINJA_subgraph as cpp_subgraph
|
| 22 | from data_lang import NINJA_subgraph as data_lang_subgraph
|
| 23 | from display import NINJA_subgraph as display_subgraph
|
| 24 | from frontend import NINJA_subgraph as frontend_subgraph
|
| 25 | from osh import NINJA_subgraph as osh_subgraph
|
| 26 | from mycpp import NINJA_subgraph as mycpp_subgraph
|
| 27 | from pea import NINJA_subgraph as pea_subgraph
|
| 28 | from prebuilt import NINJA_subgraph as prebuilt_subgraph
|
| 29 | from yaks import NINJA_subgraph as yaks_subgraph
|
| 30 | from ysh import NINJA_subgraph as ysh_subgraph
|
| 31 |
|
| 32 | from vendor import ninja_syntax
|
| 33 |
|
| 34 | # The file Ninja runs by default.
|
| 35 | BUILD_NINJA = 'build.ninja'
|
| 36 |
|
| 37 |
|
| 38 | def TarballManifest(some_files):
|
| 39 | names = []
|
| 40 |
|
| 41 | # Code we know about
|
| 42 | names.extend(some_files)
|
| 43 |
|
| 44 | names.extend([
|
| 45 | # Text
|
| 46 | 'LICENSE.txt',
|
| 47 | 'README-native.txt',
|
| 48 | 'INSTALL.txt',
|
| 49 | 'configure',
|
| 50 | 'install',
|
| 51 | 'doc/osh.1',
|
| 52 |
|
| 53 | # Build Scripts
|
| 54 | 'build/common.sh',
|
| 55 | 'build/native.sh',
|
| 56 | 'build/static-oils.sh',
|
| 57 |
|
| 58 | # These 2 are used by build/ninja-rules-cpp.sh
|
| 59 | 'build/py2.sh',
|
| 60 | 'build/dev-shell.sh',
|
| 61 | 'build/ninja-rules-cpp.sh',
|
| 62 |
|
| 63 | # These are in build/py.sh, not Ninja. Should probably put them in Ninja.
|
| 64 | #'_gen/frontend/help_meta.h',
|
| 65 | '_gen/frontend/match.re2c.h',
|
| 66 | '_gen/frontend/id_kind.asdl_c.h',
|
| 67 | '_gen/frontend/types.asdl_c.h',
|
| 68 | ])
|
| 69 |
|
| 70 | # For configure
|
| 71 | names.extend(glob('build/detect-*.c'))
|
| 72 |
|
| 73 | names.extend(glob('cpp/*.h'))
|
| 74 | names.extend(glob('mycpp/*.h'))
|
| 75 | names.extend(glob('asdl/*.h'))
|
| 76 |
|
| 77 | # ONLY the headers
|
| 78 | names.extend(glob('prebuilt/*/*.h'))
|
| 79 |
|
| 80 | names.sort() # Pass them to tar sorted
|
| 81 |
|
| 82 | # Check for dupes here
|
| 83 | unique = sorted(set(names))
|
| 84 | if names != unique:
|
| 85 | dupes = [n for n in names if names.count(n) > 1]
|
| 86 | raise AssertionError("Tarball manifest shouldn't have duplicates: %s" %
|
| 87 | dupes)
|
| 88 |
|
| 89 | for name in names:
|
| 90 | print(name)
|
| 91 |
|
| 92 |
|
| 93 | def _WinPath(s):
|
| 94 | return s.replace('/', '\\')
|
| 95 |
|
| 96 |
|
| 97 | def BatchFunctions(app_name, cc_sources, f, argv0):
|
| 98 | objects = []
|
| 99 | in_out = []
|
| 100 | for src in sorted(cc_sources):
|
| 101 | # e.g. _build/obj/cxx-dbg-sh/posix.o
|
| 102 | prefix, _ = os.path.splitext(src)
|
| 103 |
|
| 104 | #obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
|
| 105 | obj = '_build/obj/cxx-dbg/%s.o' % prefix
|
| 106 | in_out.append((src, obj))
|
| 107 |
|
| 108 | obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
|
| 109 | for d in obj_dirs:
|
| 110 | print('mkdir %s' % _WinPath(d), file=f)
|
| 111 |
|
| 112 | print('', file=f)
|
| 113 |
|
| 114 | for i, (src, obj) in enumerate(in_out):
|
| 115 | #obj_quoted = '"%s"' % obj
|
| 116 | objects.append(_WinPath(obj))
|
| 117 |
|
| 118 | # TODO: OILS_WIN32 is a hack, should have ./configure for dprintf()
|
| 119 | print('g++ -I . -D OILS_WIN32 -D MARK_SWEEP -c -o %s %s' %
|
| 120 | (_WinPath(obj), _WinPath(src)),
|
| 121 | file=f)
|
| 122 | print('if %ERRORLEVEL% neq 0 goto :error', file=f)
|
| 123 |
|
| 124 | print('', file=f)
|
| 125 |
|
| 126 | # Link
|
| 127 |
|
| 128 | out_dir = '_bin/cxx-dbg'
|
| 129 | out = '%s/%s' % (out_dir, app_name)
|
| 130 | print('mkdir %s' % _WinPath(out_dir), file=f)
|
| 131 | print('g++ -o %s %s' % (_WinPath(out), ' '.join(objects)), file=f)
|
| 132 |
|
| 133 | print('', file=f)
|
| 134 | print(':error', file=f)
|
| 135 | print(' echo ERROR %ERRORLEVEL%', file=f)
|
| 136 | print(' exit /b %ERRORLEVEL%', file=f)
|
| 137 |
|
| 138 |
|
| 139 | def ShellFunctions(app_name, cc_sources, f, argv0):
|
| 140 | """
|
| 141 | Generate a shell fragment that invokes the same function that build.ninja
|
| 142 | does
|
| 143 | """
|
| 144 | # oils_for_unix -> oils-for-unix
|
| 145 | app_name_hyphens = app_name.replace('_', '-')
|
| 146 |
|
| 147 | print('''\
|
| 148 | main() {
|
| 149 | ### Compile oils-for-unix into _bin/$compiler-$variant-sh/ (not with ninja)
|
| 150 |
|
| 151 | parse_flags "$@"
|
| 152 |
|
| 153 | # Copy into locals
|
| 154 | local compiler=$FLAG_cxx
|
| 155 | local variant=$FLAG_variant
|
| 156 | local translator=$FLAG_translator
|
| 157 | local skip_rebuild=$FLAG_skip_rebuild
|
| 158 |
|
| 159 | local out_name=%s
|
| 160 |
|
| 161 | local out_dir
|
| 162 | case $translator in
|
| 163 | mycpp)
|
| 164 | out_dir=_bin/$compiler-$variant-sh
|
| 165 | ;;
|
| 166 | *)
|
| 167 | out_dir=_bin/$compiler-$variant-sh/$translator
|
| 168 | ;;
|
| 169 | esac
|
| 170 | local out_path=$out_dir/$out_name$FLAG_suffix
|
| 171 |
|
| 172 | echo
|
| 173 | echo "$0: Building $out_name: $out_path"
|
| 174 | echo " PWD = $PWD"
|
| 175 | echo " cxx = $compiler"
|
| 176 | echo " variant = $variant"
|
| 177 | echo " translator = $translator"
|
| 178 | echo " suffix = $FLAG_suffix"
|
| 179 | if test -n "$FLAG_without_readline"; then
|
| 180 | echo " without_readline = $FLAG_without_readline"
|
| 181 | fi
|
| 182 | if test -n "$skip_rebuild"; then
|
| 183 | echo " skip_rebuild = $skip_rebuild"
|
| 184 | fi
|
| 185 |
|
| 186 | if test -n "$skip_rebuild" && test -f "$out_path"; then
|
| 187 | echo
|
| 188 | echo "$0: SKIPPING build because $out_path exists"
|
| 189 | echo
|
| 190 | return
|
| 191 | fi
|
| 192 |
|
| 193 | echo
|
| 194 | ''' % app_name_hyphens,
|
| 195 | file=f)
|
| 196 |
|
| 197 | objects = []
|
| 198 |
|
| 199 | # Filenames that depend on the translator
|
| 200 | in_out = [
|
| 201 | ('_gen/bin/%s.$translator.cc' % app_name,
|
| 202 | '_build/obj/$compiler-$variant-sh/_gen/bin/%s.$translator.o' %
|
| 203 | app_name),
|
| 204 | ('_gen/bin/%s.$translator-main.cc' % app_name,
|
| 205 | '_build/obj/$compiler-$variant-sh/_gen/bin/%s.$translator-main.o' %
|
| 206 | app_name),
|
| 207 | ]
|
| 208 | for src in sorted(cc_sources):
|
| 209 | # e.g. _build/obj/cxx-dbg-sh/posix.o
|
| 210 | prefix, _ = os.path.splitext(src)
|
| 211 |
|
| 212 | # HACK to skip the special ones above
|
| 213 | if re.match(r'.*\.mycpp.*\.cc$', src):
|
| 214 | continue
|
| 215 |
|
| 216 | obj = '_build/obj/$compiler-$variant-sh/%s.o' % prefix
|
| 217 | in_out.append((src, obj))
|
| 218 |
|
| 219 | if 0:
|
| 220 | from pprint import pformat
|
| 221 | log('cc_sources = %s', pformat(cc_sources))
|
| 222 | log('in_out = %s', pformat(in_out))
|
| 223 |
|
| 224 | bin_dir = '_bin/$compiler-$variant-sh/$translator'
|
| 225 | obj_dirs = sorted(set(os.path.dirname(obj) for _, obj in in_out))
|
| 226 |
|
| 227 | all_dirs = [bin_dir] + obj_dirs
|
| 228 | # Double quote
|
| 229 | all_dirs = ['"%s"' % d for d in all_dirs]
|
| 230 |
|
| 231 | print(' mkdir -p \\', file=f)
|
| 232 | print(' %s' % ' \\\n '.join(all_dirs), file=f)
|
| 233 | print('', file=f)
|
| 234 |
|
| 235 | do_fork = ''
|
| 236 |
|
| 237 | for i, (src, obj) in enumerate(in_out):
|
| 238 | obj_quoted = '"%s"' % obj
|
| 239 | objects.append(obj_quoted)
|
| 240 |
|
| 241 | # Only fork one translation unit that we know to be slow
|
| 242 | if src.endswith('.$translator.cc'):
|
| 243 | # There should only be one forked translation unit
|
| 244 | # It can be turned off with OILS_PARALLEL_BUILD= _build/oils
|
| 245 | assert do_fork == ''
|
| 246 | do_fork = '_do_fork=$OILS_PARALLEL_BUILD'
|
| 247 | else:
|
| 248 | do_fork = ''
|
| 249 |
|
| 250 | if do_fork:
|
| 251 | print(' # Potentially fork this translation unit with &', file=f)
|
| 252 | print(' %s \\' % do_fork, file=f)
|
| 253 | indent = ' ' if do_fork else ''
|
| 254 | print(' %s_compile_one "$compiler" "$variant" "" \\' % indent, file=f)
|
| 255 | print(' %s %s' % (src, obj_quoted), file=f)
|
| 256 | if do_fork:
|
| 257 | print(
|
| 258 | ' _do_fork= # work around bug in some versions of the dash shell',
|
| 259 | file=f)
|
| 260 | print('', file=f)
|
| 261 |
|
| 262 | print(' # wait for the translation unit before linking', file=f)
|
| 263 | print(' echo WAIT', file=f)
|
| 264 | # time -p shows any excess parallelism on 2 cores
|
| 265 | # example: oils_for_unix.mycpp.cc takes ~8 seconds longer to compile than all
|
| 266 | # other translation units combined!
|
| 267 |
|
| 268 | # Timing isn't POSIX
|
| 269 | #print(' time -p wait', file=f)
|
| 270 | print(' wait', file=f)
|
| 271 | print('', file=f)
|
| 272 |
|
| 273 | print(' echo "LINK $out_path"', file=f)
|
| 274 | # put each object on its own line, and indent by 4
|
| 275 | # note: can't have spaces in filenames
|
| 276 | print(' set -- \\', file=f)
|
| 277 | print(' %s' % (' \\\n '.join(objects)), file=f)
|
| 278 | print('', file=f)
|
| 279 |
|
| 280 | # Strip opt binary
|
| 281 | # TODO: provide a way for the user to get symbols?
|
| 282 |
|
| 283 | print('''\
|
| 284 |
|
| 285 | link "$compiler" "$variant" "" "$out_path" "$@"
|
| 286 |
|
| 287 | if test "$variant" = opt; then
|
| 288 | strip -o "$out_path.stripped" "$out_path"
|
| 289 | fi
|
| 290 |
|
| 291 | # Symlink to unstripped binary for benchmarking
|
| 292 | cd "$out_dir" # dir may have spaces
|
| 293 | for symlink in "osh$FLAG_suffix" "ysh$FLAG_suffix"; do
|
| 294 | # like ln -v, which we can't use portably
|
| 295 | echo " $symlink -> $out_name$FLAG_suffix"
|
| 296 | ln -s -f "$out_name$FLAG_suffix" $symlink
|
| 297 | done
|
| 298 | }
|
| 299 |
|
| 300 | main "$@"
|
| 301 | ''',
|
| 302 | file=f)
|
| 303 |
|
| 304 |
|
| 305 | def WritePreprocessed(n, cc_sources):
|
| 306 | # See how much input we're feeding to the compiler. Test C++ template
|
| 307 | # explosion, e.g. <unordered_map>
|
| 308 | #
|
| 309 | # Limit to {dbg,opt} so we don't generate useless rules. Invoked by
|
| 310 | # metrics/source-code.sh
|
| 311 |
|
| 312 | pre_matrix = [
|
| 313 | ('cxx', 'dbg'),
|
| 314 | ('cxx', 'opt'),
|
| 315 | ('clang', 'dbg'),
|
| 316 | ('clang', 'opt'),
|
| 317 | ]
|
| 318 | for compiler, variant in pre_matrix:
|
| 319 | preprocessed = []
|
| 320 | for src in cc_sources:
|
| 321 | # e.g. mycpp/gc_heap.cc -> _build/preprocessed/cxx-dbg/mycpp/gc_heap.cc
|
| 322 | pre = '_build/preprocessed/%s-%s/%s' % (compiler, variant, src)
|
| 323 | preprocessed.append(pre)
|
| 324 |
|
| 325 | # Summary file
|
| 326 | n.build('_build/preprocessed/%s-%s.txt' % (compiler, variant),
|
| 327 | 'line_count', preprocessed)
|
| 328 | n.newline()
|
| 329 |
|
| 330 |
|
| 331 | def InitSteps(n):
|
| 332 | """Wrappers for build/ninja-rules-*.sh
|
| 333 |
|
| 334 | Some of these are defined in mycpp/NINJA_subgraph.py. Could move them here.
|
| 335 | """
|
| 336 | #
|
| 337 | # Compiling and linking
|
| 338 | #
|
| 339 |
|
| 340 | # Preprocess one translation unit
|
| 341 | n.rule(
|
| 342 | 'preprocess',
|
| 343 | # compile_one detects the _build/preprocessed path
|
| 344 | command=
|
| 345 | 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out',
|
| 346 | description='PP $compiler $variant $more_cxx_flags $in $out')
|
| 347 | n.newline()
|
| 348 |
|
| 349 | n.rule('line_count',
|
| 350 | command='build/ninja-rules-cpp.sh line_count $out $in',
|
| 351 | description='line_count $out $in')
|
| 352 | n.newline()
|
| 353 |
|
| 354 | # Compile one translation unit
|
| 355 | n.rule(
|
| 356 | 'compile_one',
|
| 357 | command=
|
| 358 | 'build/ninja-rules-cpp.sh compile_one $compiler $variant $more_cxx_flags $in $out $out.d',
|
| 359 | depfile='$out.d',
|
| 360 | # no prefix since the compiler is the first arg
|
| 361 | description='$compiler $variant $more_cxx_flags $in $out')
|
| 362 | n.newline()
|
| 363 |
|
| 364 | # Link objects together
|
| 365 | n.rule(
|
| 366 | 'link',
|
| 367 | command=
|
| 368 | 'build/ninja-rules-cpp.sh link $compiler $variant $more_link_flags $out $in',
|
| 369 | description='LINK $compiler $variant $more_link_flags $out $in')
|
| 370 | n.newline()
|
| 371 |
|
| 372 | # 1 input and 2 outputs
|
| 373 | n.rule('strip',
|
| 374 | command='build/ninja-rules-cpp.sh strip_ $in $out',
|
| 375 | description='STRIP $in $out')
|
| 376 | n.newline()
|
| 377 |
|
| 378 | # cc_binary can have symliks
|
| 379 | n.rule('symlink',
|
| 380 | command='build/ninja-rules-cpp.sh symlink $out $symlink_val',
|
| 381 | description='SYMLINK $out $symlink_val')
|
| 382 | n.newline()
|
| 383 |
|
| 384 | #
|
| 385 | # Code generators
|
| 386 | #
|
| 387 |
|
| 388 | n.rule(
|
| 389 | 'write-shwrap',
|
| 390 | # $in must start with main program
|
| 391 | command='build/ninja-rules-py.sh write-shwrap $template $out $in',
|
| 392 | description='make-shwrap $template $out $in')
|
| 393 | n.newline()
|
| 394 |
|
| 395 | n.rule('write-main',
|
| 396 | command=
|
| 397 | 'build/ninja-rules-py.sh write-main $template $out $main_namespace',
|
| 398 | description='write-main $out $main_namespace')
|
| 399 | n.newline()
|
| 400 |
|
| 401 | # Trivial build rule, for bin/mycpp_main -> _bin/shwrap/mycpp_souffle
|
| 402 | # while adding implicit deps
|
| 403 | n.rule('cp', command='cp $in $out', description='cp $in $out')
|
| 404 | n.newline()
|
| 405 |
|
| 406 |
|
| 407 | def main(argv):
|
| 408 | try:
|
| 409 | action = argv[1]
|
| 410 | except IndexError:
|
| 411 | action = 'ninja'
|
| 412 |
|
| 413 | try:
|
| 414 | app_name = argv[2]
|
| 415 | except IndexError:
|
| 416 | app_name = 'oils_for_unix'
|
| 417 |
|
| 418 | if action == 'ninja':
|
| 419 | f = open(BUILD_NINJA, 'w')
|
| 420 | else:
|
| 421 | f = cStringIO.StringIO() # thrown away
|
| 422 |
|
| 423 | n = ninja_syntax.Writer(f)
|
| 424 | ru = ninja_lib.Rules(n)
|
| 425 |
|
| 426 | ru.comment('InitSteps()')
|
| 427 | InitSteps(n)
|
| 428 |
|
| 429 | #
|
| 430 | # Create the graph.
|
| 431 | #
|
| 432 |
|
| 433 | asdl_subgraph.NinjaGraph(ru)
|
| 434 | ru.comment('')
|
| 435 |
|
| 436 | # translate-mycpp rule
|
| 437 | mycpp_subgraph.NinjaGraph(ru)
|
| 438 | ru.comment('')
|
| 439 |
|
| 440 | bin_subgraph.NinjaGraph(ru)
|
| 441 | ru.comment('')
|
| 442 |
|
| 443 | core_subgraph.NinjaGraph(ru)
|
| 444 | ru.comment('')
|
| 445 |
|
| 446 | cpp_subgraph.NinjaGraph(ru)
|
| 447 | ru.comment('')
|
| 448 |
|
| 449 | data_lang_subgraph.NinjaGraph(ru)
|
| 450 | ru.comment('')
|
| 451 |
|
| 452 | display_subgraph.NinjaGraph(ru)
|
| 453 | ru.comment('')
|
| 454 |
|
| 455 | frontend_subgraph.NinjaGraph(ru)
|
| 456 | ru.comment('')
|
| 457 |
|
| 458 | ysh_subgraph.NinjaGraph(ru)
|
| 459 | ru.comment('')
|
| 460 |
|
| 461 | osh_subgraph.NinjaGraph(ru)
|
| 462 | ru.comment('')
|
| 463 |
|
| 464 | pea_subgraph.NinjaGraph(ru)
|
| 465 | ru.comment('')
|
| 466 |
|
| 467 | prebuilt_subgraph.NinjaGraph(ru)
|
| 468 | ru.comment('')
|
| 469 |
|
| 470 | yaks_subgraph.NinjaGraph(ru)
|
| 471 | ru.comment('')
|
| 472 |
|
| 473 | deps = ninja_lib.Deps(ru)
|
| 474 | # Materialize all the cc_binary() rules
|
| 475 | deps.WriteRules()
|
| 476 |
|
| 477 | main_cc = '_gen/bin/%s.mycpp-main.cc' % app_name
|
| 478 |
|
| 479 | # Collect sources for metrics, tarball, shell script
|
| 480 | cc_sources = deps.SourcesForBinary(main_cc)
|
| 481 |
|
| 482 | if 0:
|
| 483 | from pprint import pprint
|
| 484 | pprint(cc_sources)
|
| 485 |
|
| 486 | # TODO: could thin these out, not generate for unit tests, etc.
|
| 487 | WritePreprocessed(n, cc_sources)
|
| 488 |
|
| 489 | ru.WritePhony()
|
| 490 |
|
| 491 | n.default(['_bin/cxx-asan/osh', '_bin/cxx-asan/ysh'])
|
| 492 |
|
| 493 | if action == 'ninja':
|
| 494 | log(' (%s) -> %s (%d targets)', argv[0], BUILD_NINJA,
|
| 495 | n.num_build_targets())
|
| 496 |
|
| 497 | elif action == 'shell':
|
| 498 | ShellFunctions(app_name, cc_sources, sys.stdout, argv[0])
|
| 499 |
|
| 500 | elif action == 'batch':
|
| 501 | # TODO: write batch file
|
| 502 | BatchFunctions(app_name, cc_sources, sys.stdout, argv[0])
|
| 503 |
|
| 504 | elif action == 'tarball-manifest':
|
| 505 | names = list(cc_sources) # copy
|
| 506 |
|
| 507 | h = deps.HeadersForBinary(main_cc)
|
| 508 | names.extend(h)
|
| 509 |
|
| 510 | if app_name == 'oils_for_unix':
|
| 511 | # TODO: SourcesForBinary() and HeadersForBinary can take MULTIPLE
|
| 512 | # -main.cc
|
| 513 | names.extend([
|
| 514 | '_gen/bin/%s.mycpp-nosouffle.cc' % app_name,
|
| 515 | '_gen/bin/%s.mycpp-nosouffle-main.cc' % app_name,
|
| 516 |
|
| 517 | # TODO: remove
|
| 518 | '_gen/bin/%s.mycpp-souffle.cc' % app_name,
|
| 519 | '_gen/bin/%s.mycpp-souffle-main.cc' % app_name,
|
| 520 | ])
|
| 521 | names.append('_build/oils.sh')
|
| 522 | else:
|
| 523 | names.append('_build/%s.sh' % app_name)
|
| 524 | names.append('_build/%s.bat' % app_name)
|
| 525 |
|
| 526 | TarballManifest(names)
|
| 527 |
|
| 528 | elif action == 'app-deps':
|
| 529 | for name in cc_sources:
|
| 530 | print(name)
|
| 531 | print('---')
|
| 532 | h_sources = deps.HeadersForBinary(main_cc)
|
| 533 | for name in h_sources:
|
| 534 | print(name)
|
| 535 |
|
| 536 | else:
|
| 537 | raise RuntimeError('Invalid action %r' % action)
|
| 538 |
|
| 539 |
|
| 540 | if __name__ == '__main__':
|
| 541 | try:
|
| 542 | main(sys.argv)
|
| 543 | except RuntimeError as e:
|
| 544 | print('FATAL: %s' % e, file=sys.stderr)
|
| 545 | sys.exit(1)
|