OILS / build / dynamic-deps.sh View on Github | oils.pub

309 lines, 135 significant
1#!/usr/bin/env bash
2#
3# Calculate and filter deps of Python apps.
4#
5# Usage:
6# build/dynamic-deps.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12# Note on the cd option, "-P": Resolve symlinks in the current working
13# directory. This is needed to make `grep $REPO_ROOT...` in `repo-filter
14# (build/dynamic-deps.sh)` work. Later, `repo-filter` and related parts may be
15# rewritten using AWK to handle it better.
16REPO_ROOT=$(cd -P "$(dirname $0)/.."; pwd)
17
18readonly PY_PATH='.:vendor/'
19
20# Temporary
21readonly DIR=_build/NINJA
22
23make-egrep() {
24 # match chars until # or space, and line must be non-empty
25 gawk '
26 match($0, /([^# ]*)/, m) {
27 contents = m[0]
28 if (contents) { # skip empty lines
29 print(contents)
30 }
31 }
32 '
33}
34
35write-filters() {
36 ### Write filename filters in the egrep -f format
37
38 # For ./NINJA-config.sh to use.
39 # This style lets us add comments.
40
41 # For asdl.asdl_main and other tools
42 make-egrep >build/default.deps-filter.txt <<'EOF'
43__init__.py
44typing.py # vendor/typing.py isn't imported normally
45EOF
46
47 #
48 # DEFAULT typecheck and translate filter
49 #
50 make-egrep >build/default.typecheck-filter.txt <<'EOF'
51__init__.py
52typing.py
53
54# OrderedDict is polymorphic
55pylib/collections_.py
56
57# lots of polymorphic stuff etc.
58mycpp/mylib.py
59EOF
60
61 make-egrep >build/default.translate-filter.txt <<'EOF'
62# generated code shouldn't be translated
63_devbuild/
64_gen/
65
66asdl/py.* # pybase.py ported by hand to C++
67
68mycpp/iolib.py # Implemented in gc_iolib.{h,cC}
69mycpp/mops.py # Implemented in gc_mops.{h,cC}
70EOF
71
72 #
73 # OILS typecheck and translate filter
74 #
75 make-egrep >bin/oils_for_unix.typecheck-filter.txt <<'EOF'
76__init__.py
77typing.py
78
79# OrderedDict is polymorphic
80pylib/collections_.py
81
82# lots of polymorphic stuff etc.
83mycpp/mylib.py
84
85# TODO: move or remove these
86tools/deps.py
87tools/readlink.py
88EOF
89
90 # On top of the typecheck filter, exclude these from translation. They are
91 # not inputs to mycpp.
92
93 make-egrep >bin/oils_for_unix.translate-filter.txt <<'EOF'
94# generated code shouldn't be translated
95_devbuild/
96_gen/
97
98# definitions that are used by */*_gen.py
99.*_def\.py
100.*_spec\.py
101
102asdl/py.* # pybase.py ported by hand to C++
103
104core/py.* # pyos.py, pyutil.py ported by hand to C++
105core/optview\.py # core/optview_gen.py
106
107data_lang/py.* # pyj8.py
108
109frontend/py.*\.py # py_readline.py ported by hand to C++
110frontend/consts.py # frontend/consts_gen.py
111frontend/match.py # frontend/lexer_gen.py
112
113mycpp/iolib.py # Implemented in gc_iolib.{h,cC}
114mycpp/mops.py # Implemented in gc_mops.{h,cC}
115
116pgen2/grammar.py # These files are re-done in C++
117pgen2/pnode.py
118pgen2/token.py
119
120# should be py_path_stat.py, because it's ported by hand to C++
121pylib/path_stat.py
122
123# should be py_bool_stat.py, because it's ported by hand to C++
124osh/bool_stat.py
125EOF
126
127 wc -l */*.*-filter.txt
128}
129
130repo-filter() {
131 ### Select files from the dynamic_deps.py output
132
133 # select what's in the repo; eliminating stdlib stuff
134 # eliminate _cache for mycpp running under Python-3.10
135 grep -F -v "$REPO_ROOT/_cache" | grep -F "$REPO_ROOT" | awk '{ print $2 }'
136}
137
138exclude-files() {
139 local filter_file=$1
140 grep -E -v -f $filter_file
141}
142
143mysort() {
144 LC_ALL=C sort
145}
146
147# Code generators
148list-gen() {
149 ls */*_gen.py
150}
151
152#
153# Invocations of dynamic_deps
154#
155
156py2-manifest() {
157 local py_module=$1
158 local dir=$2
159 PYTHONPATH=$PY_PATH /usr/bin/env python2 \
160 build/dynamic_deps.py py-manifest $py_module \
161 > $dir/all-pairs.txt
162}
163
164py3-manifest() {
165 local dir=$1
166 python3 build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt
167}
168
169py-tool() {
170 local py_module=$1
171
172 local dir=$DIR/$py_module
173 mkdir -p $dir
174
175 py2-manifest $py_module $dir
176
177 cat $dir/all-pairs.txt | repo-filter | exclude-files build/default.deps-filter.txt | mysort \
178 > $dir/deps.txt
179
180 echo "DEPS $dir/deps.txt"
181}
182
183typecheck-translate() {
184 local py_module=$1
185 local typecheck_filter=${2:-}
186 local translate_filter=${3:-}
187
188 local py_rel_path=${py_module//'.'/'/'}
189 if test -z "$typecheck_filter"; then
190 local custom="$py_rel_path.typecheck-filter.txt"
191 if test -f "$custom"; then
192 typecheck_filter=$custom
193 else
194 typecheck_filter=build/default.typecheck-filter.txt
195 fi
196 fi
197
198 if test -z "$translate_filter"; then
199 local custom="$py_rel_path.translate-filter.txt"
200 if test -f "$custom"; then
201 translate_filter=$custom
202 else
203 translate_filter=build/default.translate-filter.txt
204 fi
205 fi
206
207 if false; then
208 echo " | PY $py_module"
209 echo " | TYPECHECK $typecheck_filter"
210 echo " | TRANSLATE $translate_filter"
211 fi
212
213 local dir=$DIR/$py_module
214
215 mkdir -p $dir
216
217 py2-manifest $py_module $dir
218
219 set +o errexit
220 cat $dir/all-pairs.txt | repo-filter | exclude-files $typecheck_filter | mysort \
221 > $dir/typecheck.txt
222
223 cat $dir/typecheck.txt | exclude-files $translate_filter | mysort \
224 > $dir/translate.txt
225
226 echo DEPS $dir/*
227}
228
229# mycpp and pea deps are committed to git instead of in _build/NINJA/ because
230# users might not have Python 3.10
231
232write-pea() {
233 local module='pea.pea_main'
234 local dir=prebuilt/ninja/$module
235 mkdir -p $dir
236
237 source build/dev-shell.sh # python3
238
239 # Can't use vendor/typing.py
240 py3-manifest $dir
241
242 cat $dir/all-pairs.txt | repo-filter | mysort | tee $dir/deps.txt
243
244 echo
245 echo $dir/*
246}
247
248write-mycpp() {
249 local module='mycpp.mycpp_main'
250 local dir=prebuilt/ninja/$module
251 mkdir -p $dir
252
253 if false; then
254 ( source $MYCPP_VENV/bin/activate
255 PYTHONPATH=$REPO_ROOT:$REPO_ROOT/mycpp:$MYPY_REPO maybe-our-python3 \
256 build/dynamic_deps.py py-manifest $module > $dir/all-pairs.txt
257 )
258 fi
259
260 # TODO: it would be nicer to put this at the top of the file, but we get
261 # READONLY errors.
262 source build/dev-shell.sh
263
264 py3-manifest $dir
265
266 local deps=$dir/deps.txt
267 cat $dir/all-pairs.txt \
268 | grep -v oilshell/oil_DEPS \
269 | repo-filter \
270 | exclude-files build/default.deps-filter.txt \
271 | mysort \
272 | tee $deps
273
274 echo
275 echo $dir/*
276}
277
278mycpp-example-parse() {
279 ### Manifests for mycpp/examples/parse are committed to git
280
281 local dir=$DIR/parse
282 mkdir -p $dir
283
284 py2-manifest mycpp.examples.parse $dir
285
286 local ty=mycpp/examples/parse.typecheck.txt
287 local tr=mycpp/examples/parse.translate.txt
288
289 # TODO: remove oils-for-unix
290 cat $dir/all-pairs.txt | repo-filter |
291 exclude-files build/default.typecheck-filter.txt | mysort > $ty
292
293 cat $ty | exclude-files build/default.translate-filter.txt > $tr
294
295 wc -l $ty $tr
296
297 #head $ty $tr
298}
299
300pea-hack() {
301 # Leave out help_.py for Soil
302 grep -v '_devbuild/gen/help_meta.py' $DIR/bin.oils_for_unix/typecheck.txt \
303 > pea/oils-typecheck.txt
304}
305
306# Sourced by NINJA-config.sh
307if test $(basename $0) = 'dynamic-deps.sh'; then
308 "$@"
309fi