| 1 | # NO SHEBANG because we call it directly.
|
| 2 | """
|
| 3 | dynamic_deps.py
|
| 4 |
|
| 5 | Dynamically discover Python and C modules. We import the main module and
|
| 6 | inspect sys.modules before and after. That is, we use the exact logic that the
|
| 7 | Python interpreter does.
|
| 8 |
|
| 9 | Usage:
|
| 10 | PYTHONPATH=... py_deps.py <main module>
|
| 11 |
|
| 12 | IMPORTANT: Run this script with -S so that system libraries aren't found.
|
| 13 | """
|
| 14 | from __future__ import print_function
|
| 15 |
|
| 16 | import sys
|
| 17 |
|
| 18 | OLD_MODULES = dict(sys.modules) # Make a copy
|
| 19 |
|
| 20 | import posix # Do it afterward so we don't mess up analysis.
|
| 21 |
|
| 22 | VERBOSE = False
|
| 23 | #VERBOSE = True
|
| 24 |
|
| 25 |
|
| 26 | def log(msg, *args):
|
| 27 | if not VERBOSE:
|
| 28 | return
|
| 29 | if args:
|
| 30 | msg = msg % args
|
| 31 | print('\t', msg, file=sys.stderr)
|
| 32 |
|
| 33 |
|
| 34 | def ImportMain(main_module, old_modules):
|
| 35 | """Yields (module name, absolute path) pairs."""
|
| 36 |
|
| 37 | log('Importing %r', main_module)
|
| 38 | try:
|
| 39 | __import__(main_module)
|
| 40 | except ImportError as e:
|
| 41 | log('Error importing %r with sys.path %r', main_module, sys.path)
|
| 42 | # TODO: print better error.
|
| 43 | raise
|
| 44 |
|
| 45 | new_modules = sys.modules
|
| 46 | log('After importing: %d modules', len(new_modules))
|
| 47 |
|
| 48 | for name in sorted(new_modules):
|
| 49 | if name in old_modules:
|
| 50 | continue # exclude old modules
|
| 51 |
|
| 52 | module = new_modules[name]
|
| 53 |
|
| 54 | full_path = getattr(module, '__file__', None)
|
| 55 |
|
| 56 | # For some reason, there are entries like:
|
| 57 | # 'pan.core.os': None in sys.modules. Here's a hack to get rid of them.
|
| 58 | if module is None:
|
| 59 | log('module is None: %r', name)
|
| 60 | continue
|
| 61 | # Not sure why, but some stdlib modules don't have a __file__ attribute,
|
| 62 | # e.g. "gc", "marshal", "thread". Doesn't matter for our purposes.
|
| 63 | if full_path is None:
|
| 64 | # _sre has this issue, because it's built-in
|
| 65 | log('full_path is None: %r', name)
|
| 66 | continue
|
| 67 | yield name, full_path
|
| 68 |
|
| 69 | # Special case for __future__. It's necessary, but doesn't get counted
|
| 70 | # because we import it first!
|
| 71 | module = sys.modules['__future__']
|
| 72 | full_path = getattr(module, '__file__', None)
|
| 73 | yield '__future__', full_path
|
| 74 |
|
| 75 |
|
| 76 | PY_MODULE = 0
|
| 77 | C_MODULE = 1
|
| 78 |
|
| 79 |
|
| 80 | def FilterModules(modules):
|
| 81 | """Look at __file__ of each module, and classify them as Python or C."""
|
| 82 |
|
| 83 | for module, full_path in modules:
|
| 84 | #log('FilterModules %s %s', module, full_path)
|
| 85 | num_parts = module.count('.') + 1
|
| 86 | i = len(full_path)
|
| 87 | # Do it once more in this case
|
| 88 | if full_path.endswith('/__init__.pyc') or \
|
| 89 | full_path.endswith('__init__.py'):
|
| 90 | i = full_path.rfind('/', 0, i)
|
| 91 | for _ in range(num_parts): # range for Python 3
|
| 92 | i = full_path.rfind('/', 0, i)
|
| 93 | #print i, full_path[i+1:]
|
| 94 | rel_path = full_path[i + 1:]
|
| 95 |
|
| 96 | # Depending on whether it's cached, the __file__ attribute on the module
|
| 97 | # ends with '.py' or '.pyc'.
|
| 98 | if full_path.endswith('.py'):
|
| 99 | yield PY_MODULE, full_path, rel_path
|
| 100 | elif full_path.endswith('.pyc'):
|
| 101 | yield PY_MODULE, full_path[:-1], rel_path[:-1]
|
| 102 | else:
|
| 103 | # .so file
|
| 104 | yield C_MODULE, module, full_path
|
| 105 |
|
| 106 |
|
| 107 | def main(argv):
|
| 108 | """Returns an exit code."""
|
| 109 |
|
| 110 | # Set an environment variable so dependencies in debug mode can be excluded.
|
| 111 | posix.environ['_OVM_DEPS'] = '1'
|
| 112 |
|
| 113 | action = argv[1]
|
| 114 | main_module = argv[2]
|
| 115 | log('Before importing: %d modules', len(OLD_MODULES))
|
| 116 | log('OLD %s', OLD_MODULES.keys())
|
| 117 |
|
| 118 | if action == 'both': # Write files for both .py and .so dependencies
|
| 119 | prefix = argv[3]
|
| 120 | py_out_path = prefix + '-cpython.txt'
|
| 121 | c_out_path = prefix + '-c.txt'
|
| 122 |
|
| 123 | modules = ImportMain(main_module, OLD_MODULES)
|
| 124 | #log('NEW %s', list(modules))
|
| 125 |
|
| 126 | with open(py_out_path, 'w') as py_out, open(c_out_path, 'w') as c_out:
|
| 127 | for mod_type, x, y in FilterModules(modules):
|
| 128 | if mod_type == PY_MODULE:
|
| 129 | print(x, y, file=py_out)
|
| 130 | print(x + 'c', y + 'c',
|
| 131 | file=py_out) # .pyc goes in bytecode.zip too
|
| 132 |
|
| 133 | elif mod_type == C_MODULE:
|
| 134 | print(x, y, file=c_out) # mod_name, full_path
|
| 135 |
|
| 136 | else:
|
| 137 | raise AssertionError(mod_type)
|
| 138 |
|
| 139 | elif action == 'py': # .py path -> .pyc relative path
|
| 140 | modules = ImportMain(main_module, OLD_MODULES)
|
| 141 | for mod_type, full_path, rel_path in FilterModules(modules):
|
| 142 | if mod_type == PY_MODULE:
|
| 143 | opy_input = full_path
|
| 144 | opy_output = rel_path + 'c' # output is .pyc
|
| 145 | print(opy_input, opy_output)
|
| 146 |
|
| 147 | elif action == 'py-manifest': # .py path -> .py relative path
|
| 148 | modules = ImportMain(main_module, OLD_MODULES)
|
| 149 | for mod_type, full_path, rel_path in FilterModules(modules):
|
| 150 | if mod_type == PY_MODULE:
|
| 151 | opy_input = full_path
|
| 152 | assert rel_path.endswith('.py')
|
| 153 | #mod_name = rel_path[:-3].replace('/', '.')
|
| 154 | print(opy_input, rel_path)
|
| 155 | else:
|
| 156 | raise RuntimeError('Invalid action %r' % action)
|
| 157 |
|
| 158 |
|
| 159 | if __name__ == '__main__':
|
| 160 | try:
|
| 161 | sys.exit(main(sys.argv))
|
| 162 | except RuntimeError as e:
|
| 163 | print('%s: %s' % (sys.argv[0], e.args[0]), file=sys.stderr)
|
| 164 | sys.exit(1)
|
| 165 |
|
| 166 | # vim: ts=2
|