OILS / build / ref / ovm-compile.sh View on Github | oils.pub

430 lines, 168 significant
1#!/usr/bin/env bash
2#
3# Compile OVM tarball.
4#
5# Usage:
6# build/ovm-compile.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11shopt -s strict:all 2>/dev/null || true # dogfood for OSH
12
13REPO_ROOT=$(cd $(dirname $0)/../..; pwd)
14readonly REPO_ROOT
15
16source build/common.sh
17
18source-detected-config-or-die() {
19 if ! source _build/detected-config.sh; then
20 # Make this error stand out.
21 echo
22 echo "FATAL: can't find _build/detected-config.h. Run './configure'"
23 echo
24 exit 1
25 fi
26}
27
28# NOTES on trying to delete certain modules:
29#
30# _warnings.c: There weren't that many; it probably could be deleted.
31# bufferobject.c: the types.py module uses it.
32# Python-ast.h: pythonrun.c uses it in several places (mod_ty), and a lot of
33# stuff uses pythonrun.c.
34# pythonrun.c: lots interpreter flags and interpreter initialization caused
35# link errors.
36# pyctype.c: Tables needed for many string operations.
37
38# getargs.c: needed for Python-C API, e.g. PyArg_ParseTuple.
39# dtoa.c: not tried, but I assume that %.3f for 'time' uses it.
40
41
42readonly OVM_PYTHON_OBJS='
43Python/_warnings.c
44Python/bltinmodule.c
45Python/ceval.c
46Python/errors.c
47Python/getargs.c
48Python/getcompiler.c
49Python/getplatform.c
50Python/getversion.c
51Python/import.c
52Python/marshal.c
53Python/modsupport.c
54Python/mystrtoul.c
55Python/mysnprintf.c
56Python/pyarena.c
57Python/pyctype.c
58Python/pyfpe.c
59Python/pystate.c
60Python/pythonrun.c
61Python/random.c
62Python/structmember.c
63Python/sysmodule.c
64Python/traceback.c
65Python/pystrtod.c
66Python/dtoa.c
67Python/pymath.c
68'
69# NOTE: pystrtod.c needs some floating point functions in pymath.c
70
71OBJECT_OBJS='
72Objects/abstract.c
73Objects/boolobject.c
74Objects/bufferobject.c
75Objects/bytes_methods.c
76Objects/capsule.c
77Objects/cellobject.c
78Objects/classobject.c
79Objects/cobject.c
80Objects/codeobject.c
81Objects/descrobject.c
82Objects/enumobject.c
83Objects/exceptions.c
84Objects/genobject.c
85Objects/fileobject.c
86Objects/floatobject.c
87Objects/frameobject.c
88Objects/funcobject.c
89Objects/intobject.c
90Objects/iterobject.c
91Objects/listobject.c
92Objects/longobject.c
93Objects/dictobject.c
94Objects/methodobject.c
95Objects/moduleobject.c
96Objects/object.c
97Objects/obmalloc.c
98Objects/rangeobject.c
99Objects/setobject.c
100Objects/sliceobject.c
101Objects/stringobject.c
102Objects/structseq.c
103Objects/tupleobject.c
104Objects/typeobject.c
105Objects/weakrefobject.c
106'
107
108# Non-standard lib stuff.
109MODULE_OBJS='
110Modules/main.c
111Modules/gcmodule.c
112'
113
114# The stuff in Modules/Setup.dist, signalmodule.c. NOTE: In Python,
115# signalmodule.c is specified in Modules/Setup.config, which comes from
116# 'configure' output.
117MODOBJS='
118Modules/errnomodule.c
119Modules/pwdmodule.c
120Modules/_sre.c
121Modules/_weakref.c
122Modules/zipimport.c
123Modules/signalmodule.c
124'
125
126# Parser/myreadline.c is needed for raw_input() to work. There is a dependency
127# from Python/bltinmodule.c to it.
128OVM_LIBRARY_OBJS="
129Modules/getbuildinfo.c
130Parser/myreadline.c
131$OBJECT_OBJS
132$OVM_PYTHON_OBJS
133$MODULE_OBJS
134$MODOBJS
135"
136
137readonly EMPTY_STR='""'
138
139# Stub out a few variables
140readonly PREPROC_FLAGS=(
141 -D OVM_MAIN \
142 -D PYTHONPATH="$EMPTY_STR" \
143 -D VERSION="$EMPTY_STR" \
144 -D VPATH="$EMPTY_STR" \
145 -D Py_BUILD_CORE \
146 # Python already has support for disabling complex numbers!
147 -D WITHOUT_COMPLEX
148)
149
150readonly INCLUDE_PATHS=(
151 -I . # for pyconfig.h
152 -I .. # for _gen/frontend/id_kind_asdl_c.h etc.
153 -I Include
154)
155readonly CC=${CC:-cc} # cc should be on POSIX systems
156
157# BASE_CFLAGS is copied by observation from what configure.ac does on my Ubuntu
158# 16.04 system. Then we check if it works on Alpine Linux too.
159
160# "Python violates C99 rules, by casting between incompatible pointer types.
161# GCC may generate bad code as a result of that, so use -fno-strict-aliasing if
162# supported."
163# - gcc 4.x and Clang need -fwrapv
164
165# TODO:
166# - -DNDEBUG is also passed. That turns off asserts. Do we want that?
167# - We should auto-detect the flags in configure, or simplify the source so it
168# isn't necessary. Python's configure.ac sometimes does it by compiling a test
169# file; at other times it does it by grepping $CC --help.
170
171# pyext/fanos.c needs -std=c99
172BASE_CFLAGS='-fno-strict-aliasing -fwrapv -Wall -Wstrict-prototypes -std=c99'
173
174# These flags are disabled for OS X. I would have thought it would work in
175# Clang? It works with both GCC and Clang on Linux.
176# https://stackoverflow.com/questions/6687630/how-to-remove-unused-c-c-symbols-with-gcc-and-ld
177#BASE_CFLAGS="$BASE_CFLAGS -fdata-sections -ffunction-sections"
178
179# Needed after cpython-defs filtering.
180BASE_CFLAGS="$BASE_CFLAGS -Wno-unused-variable -Wno-unused-function"
181readonly BASE_CFLAGS
182
183BASE_LDFLAGS=''
184# Disabled for OS X
185# BASE_LDFLAGS='-Wl,--gc-sections'
186
187# The user should be able to customize CFLAGS, but it shouldn't disable what's
188# in BASE_CFLAGS.
189readonly CFLAGS=${CFLAGS:-}
190readonly LDFLAGS=${LDFLAGS:-}
191
192build() {
193 local out=${1:-$PY27/ovm2}
194 local module_init=${2:-$PY27/Modules/config.c}
195 local main_name=${3:-_tmp/hello/main_name.c}
196 local c_module_srcs=${4:-_tmp/hello/c-module-srcs.txt}
197 shift 4
198
199 local abs_out=$PWD/$out
200 local abs_module_init=$PWD/$module_init
201 local abs_main_name=$PWD/$main_name
202 local abs_c_module_srcs=$PWD/$c_module_srcs
203
204 #echo $OVM_LIBRARY_OBJS
205
206 # HAVE_READLINE defined in detected-config.sh.
207 source-detected-config-or-die
208
209 pushd $PY27
210
211 local readline_flags=''
212 if [[ "$HAVE_READLINE" -eq 1 ]]; then
213 # Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c.
214 # For now, we are using raw_input() for the REPL. TODO: Parameterize this!
215 # We should create a special no_readline_raw_input().
216
217 c_module_src_list=$(cat $abs_c_module_srcs)
218
219 if [[ -n "$READLINE_DIR" ]]; then
220 readline_flags+="-L $READLINE_DIR/lib -I $READLINE_DIR/include "
221 fi
222
223 # NOTE: pyconfig.h has HAVE_LIBREADLINE but doesn't appear to use it?
224 readline_flags+="-l readline -D HAVE_READLINE"
225 else
226 # don't fail
227 c_module_src_list=$(grep -E -v '/readline.c|/line_input.c' $abs_c_module_srcs || true)
228 fi
229
230 # $PREFIX comes from ./configure and defaults to /usr/local.
231 # $EXEC_PREFIX is a GNU thing and used in getpath.c. Could probably get rid
232 # of it.
233
234 time $CC \
235 ${BASE_CFLAGS} \
236 ${CFLAGS} \
237 "${INCLUDE_PATHS[@]}" \
238 "${PREPROC_FLAGS[@]}" \
239 -D PREFIX="\"$PREFIX\"" \
240 -D EXEC_PREFIX="\"$PREFIX\"" \
241 -o $abs_out \
242 $OVM_LIBRARY_OBJS \
243 $abs_module_init \
244 $abs_main_name \
245 $c_module_src_list \
246 Modules/ovm.c \
247 -l m \
248 ${BASE_LDFLAGS} \
249 ${LDFLAGS} \
250 $readline_flags \
251 "$@"
252
253 # NOTE:
254 # -l readline -l termcap -- for Python readline. Hm it builds without -l
255 # termcap.
256 # -l z WOULD be needed for zlibmodule.c, but we don't need it because our zip
257 # file has no compression -- see build/ref/make_zip.py with ZIP_STORED.
258 # zipimport works fine without this.
259}
260
261# build the optimized one. Makefile uses -O3.
262
263# Clang -O2 is 1.37 MB. 18 seconds to compile.
264# -m32 is 1.12 MB. But I probably have to redefine a few things because
265# there are more warnings.
266# -O3 is 1.40 MB.
267
268# GCC -O2 is 1.35 MB. 21 seconds to compile.
269
270build-dbg() {
271 build "$@" -O0 -g -D OVM_DEBUG
272}
273
274# This will be stripped later.
275build-opt() {
276 # frame pointer for perf. Otherwise stack traces are messed up!
277 # http://www.brendangregg.com/FlameGraphs/cpuflamegraphs.html#C But why
278 # isn't debuginfo enough? Because it's a recursive function?
279 # Does this make things slower? Do I need a "perf" build?
280 build "$@" -O3 -fno-omit-frame-pointer
281}
282
283#
284# Source Release (uses same files
285#
286
287add-py27() {
288 xargs -I {} -- echo $PY27/{}
289}
290
291python-sources() {
292 echo "$OVM_LIBRARY_OBJS" | add-py27
293}
294
295_headers() {
296 local c_module_srcs=${1:-_tmp/hello/c-module-srcs.txt}
297 local abs_c_module_srcs=$PWD/$c_module_srcs
298
299 cd $PY27
300
301 # NOTE: build/py.sh configure-for-dev skips OVM tarball stuff, which may
302 # cause spurious LONG_BIT errors here. They are harmless
303
304 # -MM: no system headers
305 gcc \
306 "${INCLUDE_PATHS[@]}" \
307 "${PREPROC_FLAGS[@]}" \
308 -MM \
309 $OVM_LIBRARY_OBJS \
310 Modules/ovm.c \
311 $(cat $abs_c_module_srcs)
312}
313
314# NOTE: 91 headers in Include, but only 81 referenced here. So it's worth it.
315# These are probably for the parser.
316#
317# NOTE: We also should get rid of asdl.h and so forth.
318
319python-headers() {
320 local c_module_srcs=$1
321
322 # 1. -MM outputs Makefile fragments, so egrep turns those into proper lines.
323 #
324 # 2. The user should generate detected-config.h, so remove it.
325 #
326 # 3. # gcc outputs paths like
327 # Python-2.7.13/Python/../Objects/stringlib/stringdefs.h
328 # but 'Python/..' causes problems for tar.
329 #
330
331 _headers $c_module_srcs \
332 | egrep --only-matching '[^ ]+\.h' \
333 | grep -v '_build/detected-config.h' \
334 | sed 's|^Python/../||' \
335 | sort | uniq | add-py27
336}
337
338make-tar() {
339 local app_name=${1:-hello}
340 local bytecode_zip=${2:-bytecode-cpython.zip}
341 local out=${3:-_release/hello.tar}
342
343 local version_file
344 case $app_name in
345 oils-ref)
346 version_file=oils-version.txt
347 ;;
348 hello)
349 version_file=build/testdata/hello-version.txt
350 ;;
351 *)
352 die "Unknown app $app_name"
353 exit 1
354 ;;
355 esac
356 local version=$(head -n 1 $version_file)
357
358 echo "Creating $app_name version $version"
359
360 local c_module_srcs=_build/$app_name/c-module-srcs.txt
361
362 # Add oils-ref-0.0.0/ to the beginning of every path.
363 local sed_expr="s,^,${app_name}-${version}/,"
364
365 # Differences between tarball and repo:
366 #
367 # - build/portable-rules.mk is intentionally not included in the release tarball.
368 # The Makefile can and should operate without it.
369 #
370 # - We include intermediate files like c-module-srcs.txt, so we don't have to
371 # ship tools dynamic_deps.py. The end-user build shouldn't depend on Python.
372
373 # Note: python-headers runs gcc -M, including pyconfig.h and
374 # _build/detected-config.h.
375
376 tar --create --transform "$sed_expr" --file $out \
377 LICENSE.txt \
378 INSTALL-old.txt \
379 configure \
380 install \
381 uninstall \
382 Makefile \
383 doc/osh.1 \
384 build/ref/ovm-compile.sh \
385 build/ref/ovm-actions.sh \
386 build/clean.sh \
387 build/common.sh \
388 build/detect-*.c \
389 _build/$app_name/$bytecode_zip \
390 _build/$app_name/*.c \
391 $PY27/LICENSE \
392 $PY27/Modules/ovm.c \
393 $c_module_srcs \
394 $(cat $c_module_srcs | add-py27) \
395 $(python-headers $c_module_srcs) \
396 $(python-sources)
397
398 ls -l $out
399}
400
401# 123K lines.
402# Excluding MODOBJS, it's 104K lines.
403#
404# Biggest: posixmodule,unicodeobject,typeobject,ceval.
405#
406# Remove tmpnam from posixmodule, other cruft.
407#
408# Big ones to rid of: unicodeobject.c, import.c
409# codecs and codecsmodule? There is some non-unicode stuff there though.
410#
411# Probably need unicode for compatibility with modules and web frameworks
412# especially.
413
414count-c-lines() {
415 pushd $PY27
416 wc -l $OVM_LIBRARY_OBJS | sort -n
417
418 # 90 files.
419 # NOTE: To count headers, use the tar file.
420 echo
421 echo 'Files:'
422 { for i in $OVM_LIBRARY_OBJS; do
423 echo $i
424 done
425 } | wc -l
426
427 popd
428}
429
430"$@"