OILS / metrics / source-code.sh View on Github | oils.pub

627 lines, 341 significant
1#!/usr/bin/env bash
2#
3# Count lines of code in various ways.
4#
5# Usage:
6# metrics/source-code.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
13readonly REPO_ROOT
14
15source test/common.sh
16source test/tsv-lib.sh
17
18filter-py() {
19 grep -E -v '__init__.py$|_gen.py|_test.py|_tests.py|NINJA_subgraph.py$'
20}
21
22readonly -a OSH_ASDL=( {frontend,core,display}/*.asdl )
23
24oils-files() {
25 # what's in the runtime
26 osh-files
27 ysh-files
28 data-lang-files
29 tools-files
30}
31
32# OSH and common
33osh-files() {
34 # Exclude:
35 # - line_input.c because I didn't write it. It still should be minimized.
36 # - code generators
37 # - test library
38 #
39 # note: could move display/ to a separate part
40 ls bin/oils_for_unix.py {osh,core,display,frontend}/*.py builtin/*_osh.py \
41 pyext/*.c */*.pyi \
42 "${OSH_ASDL[@]}" \
43 | filter-py | grep -E -v 'posixmodule.c$|line_input.c$|_gen.py$|test_lib.py$|os.pyi$'
44}
45
46# cloc doesn't understand ASDL files.
47# Use a wc-like format, filtering out blank lines and comments.
48asdl-cloc() {
49 python -c '
50import sys
51
52total = 0
53for path in sys.argv[1:]:
54 num_lines = 0
55 with open(path) as f:
56 for line in f:
57 line = line.strip()
58 if not line or line.startswith("#"):
59 continue
60 num_lines += 1
61
62 print "%5d %s" % (num_lines, path)
63 total += num_lines
64
65print "%5d %s" % (total, "total")
66' "$@"
67}
68
69cloc-report() {
70 echo '(non-blank non-comment lines)'
71 echo
72
73 echo 'OSH'
74 echo
75 osh-files | xargs cloc --quiet "$@"
76 echo
77 echo
78
79 echo 'YSH'
80 echo
81 ysh-files | xargs cloc --quiet "$@"
82 echo
83 echo
84
85 echo 'Data Languages'
86 echo
87 data-lang-files | xargs cloc --quiet "$@"
88 echo
89 echo
90
91 echo 'Tools'
92 echo
93 tools-files | xargs cloc --quiet "$@"
94 echo
95 echo
96
97 echo 'ASDL SCHEMAS (non-blank non-comment lines)'
98 asdl-cloc "${OSH_ASDL[@]}" data_lang/*.asdl
99 echo
100 echo
101
102 # NOTE: --csv option could be parsed into HTML.
103 # Or just sum with asdl-cloc!
104
105 echo 'Hand-Written C++ code (non-blank non-comment lines)'
106 echo
107 { cpp-binding-files; mycpp-runtime-files; } | xargs cloc --quiet "$@"
108}
109
110preprocessed() {
111 ./NINJA-config.sh
112
113 # Clang has slightly fewer lines, but it's not on the CI machine
114 #local -a files=(_build/preprocessed/{cxx,clang}-{dbg,opt}.txt)
115
116 local -a files=(_build/preprocessed/cxx-{dbg,opt}.txt)
117
118 ninja "${files[@]}"
119
120 # Publish with release and show and CI
121
122 local dir=_tmp/metrics/preprocessed
123 mkdir -p $dir
124 cp -v "${files[@]}" $dir
125
126 head -n 100 $dir/*.txt
127}
128
129#
130# Two variants of the $count function: text and html
131#
132
133category-text() {
134 local header=$1
135 local comment=$2
136
137 echo "$header"
138 # omit comment
139
140 # stdin is the files
141 xargs wc -l | sort --numeric
142 echo
143}
144
145# This is overly clever ...
146shopt -s lastpipe
147SECTION_ID=0 # mutable global
148
149category-html() {
150 # TODO: Don't use wc -l, and just count and sum the lines yourself
151
152 xargs wc -l | metrics/line_counts.py $((++SECTION_ID)) "$@"
153}
154
155#
156# Functions That Count
157#
158
159# Note this style is OVERLY ABSTRACT, but it's hard to do better in shell. We
160# want to parameterize over text and HTML. In YSH I think we would use this:
161#
162# proc p1 {
163# category 'OSH (and common libraries)' {
164# comment = 'This is the input'
165# osh-files | read --lines :files
166# }
167# }
168#
169# This produces a series of dicts that looks like
170# { name: 'OSH ...', comment: "This ...", files: %(one two three) }
171#
172# Then we iterate over the categories and produce text or HTML.
173
174osh-counts() {
175 local count=$1
176 shift
177
178 osh-files | $count \
179 'OSH (and common libraries)' \
180 'This is the input to the translators, written in statically-typed Python. Note that bash is at least 140K lines of code, and OSH implements a large part of bash and more.' \
181 "$@"
182}
183
184ysh-files() {
185 # Count meta_oils.py as YSH, not OSH, even though it contains the shell
186 # 'builtin command type' builtins. We will generalize that a bit
187 ls ysh/*.{py,pgen2} \
188 builtin/{func,method}*.py \
189 builtin/*_ysh.py \
190 builtin/*_oils.py \
191 | filter-py
192}
193
194ysh-counts() {
195 local count=$1
196 shift
197
198 ysh-files | $count \
199 'YSH' 'Expression grammar, parser, evaluator, etc.' "$@"
200}
201
202data-lang-files() {
203 ls data_lang/*.asdl
204 ls data_lang/*.py | filter-py
205 ls data_lang/*.{c,h} | egrep -v '_test' # exclude j8_test_lib as well
206}
207
208data-lang-counts() {
209 local count=$1
210 shift
211
212 data-lang-files | $count \
213 'Data Languages' 'JSON, J8 Notation, ...' "$@"
214}
215
216tools-files() {
217 ls tools/*.py | filter-py
218}
219
220tools-counts() {
221 local count=$1
222 shift
223
224 tools-files | $count \
225 'Tools' '' "$@"
226}
227
228cpp-binding-files() {
229 ls cpp/*.{cc,h} | egrep -v '_test.cc'
230}
231
232mycpp-runtime-files() {
233 ls mycpp/*.{cc,h} | egrep -v '_test.cc|bump_leak_heap'
234}
235
236cpp-counts() {
237 local count=$1
238 shift
239
240 cpp-binding-files | $count \
241 'Hand-written C++ Code' \
242 'Includes OS bindings. Small C++ files like cpp/osh_arith_parse.{cc,h} correspond to larger Python files like osh/arith_parse.py.' \
243 "$@"
244
245 # Remove code that isn't "in production"
246 mycpp-runtime-files | $count \
247 'Garbage-Collected Runtime' \
248 'Uses a fork-friendly Mark-Sweep collector.' \
249 "$@"
250
251 ls mycpp/*_test.cc cpp/*_test.cc | $count \
252 'Unit tests in C++' \
253 'The goal is to make the spec tests pass, but unit tests are helpful too.' \
254 "$@"
255
256 ls NINJA*.sh */NINJA*.py build/ninja*.{sh,py} | $count \
257 'Incremental C++ Build' '' "$@"
258}
259
260gen-cpp-counts() {
261 local count=$1
262 shift
263
264 # NOTE: this excludes .re2c.h file
265 ls _gen/*/*.{cc,h} | $count \
266 'Generated C++ Code' \
267 'mycpp generates the big file _gen/bin/oils-for-unix.mycpp.cc. Other programs like Zephyr ASDL and re2c generate other files.' \
268 "$@"
269}
270
271mycpp-translator-files() {
272 # compare_pairs is for testing mycpp-examples
273 ls mycpp/*.py | egrep -v 'mylib|iolib|mops|compare_pairs'
274}
275
276mycpp-counts() {
277 local count=$1
278 shift
279
280 ls mycpp/{mylib,iolib,mops}.py | grep -v 'NINJA_subgraph.py' | filter-py | $count \
281 'mycpp Python Runtime' \
282 "Stubs that are re-implemented in C++" \
283 "$@"
284
285 mycpp-translator-files | grep -v 'NINJA_subgraph.py' | filter-py | $count \
286 'mycpp Translator' \
287 "This prototype uses the MyPy frontend to translate statically-typed Python to C++. The generated code calls a small runtime which implements things like List[T], Dict[K, V], and Python's len()." \
288 "$@"
289
290 ls mycpp/examples/*.py | $count \
291 'mycpp Test Data' \
292 'Small Python examples that translate to C++, compile, and run.' \
293 "$@"
294}
295
296code-generator-counts() {
297 local count=$1
298 shift
299
300 ls asdl/*.py | filter-py | grep -v -E 'arith_|tdop|_demo' | $count \
301 'Zephyr ASDL' \
302 'A DSL for algebraic data types, borrowed from Python. Oils is the most strongly typed Bourne shell implementation!' \
303 "$@"
304
305 ls pgen2/*.py | filter-py | $count \
306 'pgen2 Parser Generator' \
307 'An LL(1) parser generator used to parse YSH expressions. Also borrowed from CPython.' \
308 "$@"
309
310 ls */*_gen.py | $count \
311 'Other Code Generators' \
312 'In order to make Oils statically typed, we had to abandon Python reflection and use C++ source code generation instead. The lexer, flag definitions, and constants can be easily compiled to C++.' \
313 "$@"
314
315 ls yaks/*.py | filter-py | $count \
316 'Yaks' \
317 'Experimental replacement for mycpp' \
318 "$@"
319}
320
321spec-gold-counts() {
322 local count=$1
323 shift
324
325 ls spec/*.test.sh | $count \
326 'Spec Tests' \
327 'A comprehensive test suite that compares OSH against other shells. If OSH passes these tests in BOTH Python and C++, it means that the translation works.' \
328 "$@"
329
330 ls test/gold/*.sh | $count \
331 'Gold Tests' \
332 'Another suite that tests shells "from the outside". Instead of making explicit assertions, we verify that OSH behaves like bash.' \
333 "$@"
334}
335
336#
337# Top Level Summaries
338#
339
340_for-translation() {
341 local count=$1
342 shift
343
344 mycpp-counts $count "$@"
345
346 code-generator-counts $count "$@"
347
348 cpp-counts $count "$@"
349
350 osh-counts $count "$@"
351
352 ysh-counts $count "$@"
353
354 data-lang-counts $count "$@"
355
356 tools-counts $count "$@"
357
358 spec-gold-counts $count "$@"
359
360 gen-cpp-counts $count "$@"
361}
362
363_overview() {
364 local count=$1
365 shift
366
367 osh-counts $count "$@"
368
369 ysh-counts $count "$@"
370
371 data-lang-counts $count "$@"
372
373 tools-counts $count "$@"
374
375 ls stdlib/osh/*.sh | $count \
376 "OSH stdlib" '' "$@"
377
378 ls stdlib/ysh/*.ysh | $count \
379 "YSH stdlib" '' "$@"
380
381 ls pylib/*.py | filter-py | $count \
382 "Code Borrowed from Python's stdlib" '' "$@"
383
384 spec-gold-counts $count "$@"
385
386 test/unit.sh files-to-count | $count \
387 'Python Unit Tests' '' "$@"
388
389 ls test/*.{sh,py,R} | filter-py | grep -v jsontemplate.py | $count \
390 'Other Shell Tests' '' "$@"
391
392 ls */TEST.sh | $count \
393 'Test Automation' '' "$@"
394
395 mycpp-counts $count "$@"
396
397 code-generator-counts $count "$@"
398
399 cpp-counts $count "$@"
400
401 # Leaving off gen-cpp-counts since that requires a C++ build
402
403 ls build/*.{sh,py,c} configure install \
404 | filter-py | egrep -v 'NINJA|TEST' | $count \
405 'Build Automation' '' "$@"
406
407 ls build/ref/*.{mk,sh,py} Makefile \
408 | filter-py | $count \
409 'Build of oils-ref Tarball' '' "$@"
410
411 ls devtools/release*.sh | $count \
412 'Release Automation' '' "$@"
413
414 ls soil/*.{sh,py} | $count \
415 'Soil: Multi-cloud CI with containers' '' "$@"
416
417 ls benchmarks/*.{sh,py,R} | $count \
418 'Benchmarks' '' "$@"
419
420 ls metrics/*.{sh,R} | $count \
421 'Metrics' '' "$@"
422
423 ls _devbuild/gen/*.py | $count \
424 'Generated Python Code' \
425 'For the Python App Bundle.' \
426 "$@"
427
428 ls doctools/*.py doctools/*.{h,cc} | filter-py | $count \
429 'Doc Tools' '' "$@"
430
431 ls web/*.js web/*/*.{js,py} | $count \
432 'Web' '' "$@"
433}
434
435for-translation() {
436 _for-translation category-text
437}
438
439overview() {
440 _overview category-text
441}
442
443print-files() {
444 xargs -n 1 -- echo
445}
446
447overview-list() {
448 _overview print-files
449}
450
451#
452# HTML Versions
453#
454
455html-head() {
456 PYTHONPATH=. doctools/html_head.py "$@"
457}
458
459metrics-html-head() {
460 local title="$1"
461
462 local base_url='../../../web'
463
464 html-head --title "$title" "$base_url/base.css" "$base_url/table/table-sort.css" "$base_url/line-counts.css"
465}
466
467counts-html() {
468 local name=$1
469 local title=$2
470
471 local tmp_dir=_tmp/metrics/line-counts/$name
472
473 rm -r -f -v $tmp_dir >& 2
474 mkdir -v -p $tmp_dir >& 2
475
476 tsv-row category category_HREF total_lines num_files > $tmp_dir/INDEX.tsv
477
478 echo $'column_name\ttype
479category\tstring
480category_HREF\tstring
481total_lines\tinteger
482num_files\tinteger' >$tmp_dir/INDEX.schema.tsv
483
484 # Generate the HTML
485 "_$name" category-html $tmp_dir
486
487 metrics-html-head "$title"
488 echo ' <body class="width40">'
489
490 echo "<h1>$title</h1>"
491
492 tsv2html $tmp_dir/INDEX.tsv
493
494 echo '<hr/>'
495
496 # All the parts
497 cat $tmp_dir/*.html
498
499 echo ' </body>'
500 echo '</html>'
501}
502
503for-translation-html() {
504 local title='Overview: Translating Oils to C++'
505 counts-html for-translation "$title"
506}
507
508overview-html() {
509 local title='Overview of Oils Code'
510 counts-html overview "$title"
511}
512
513write-reports() {
514 local out_dir=${1:-_tmp/metrics/line-counts}
515
516 mkdir -v -p $out_dir
517
518 for-translation-html > $out_dir/for-translation.html
519
520 overview-html > $out_dir/overview.html
521
522 ls -l $out_dir
523}
524
525#
526# Misc
527#
528
529# count instructions, for fun
530instructions() {
531 # http://pepijndevos.nl/2016/08/24/x86-instruction-distribution.html
532
533 local bin=_build/oil/ovm-opt.stripped
534 objdump -d $bin | cut -f3 | grep -oE "^[a-z]+" | hist
535}
536
537hist() {
538 sort | uniq -c | sort -n
539}
540
541stdlib-imports() {
542 oil-osh-files | xargs grep --no-filename '^import' | hist
543}
544
545imports() {
546 oil-osh-files | xargs grep --no-filename -w import | hist
547}
548
549imports-not-at-top() {
550 oil-osh-files | xargs grep -n -w import | awk -F : ' $2 > 100'
551}
552
553# For the compiler, see what's at the top level.
554top-level() {
555 grep '^[a-zA-Z]' {core,osh}/*.py \
556 | grep -v '_test.py' \
557 | egrep -v ':import|from|class|def' # note: colon is from grep output
558}
559
560_python-symbols() {
561 local main=$1
562 local name=$2
563 local out_dir=$3
564
565 mkdir -p $out_dir
566 local out=${out_dir}/${name}-symbols.txt
567
568 # To debug what version we're running eci
569 /usr/bin/env python2 -V
570 echo
571
572 # Run this from the repository root.
573 PYTHONPATH='.:vendor/' CALLGRAPH=1 $main | tee $out
574
575 wc -l $out
576 echo
577 echo "Wrote $out"
578}
579
580oil-python-symbols() {
581 local out_dir=${1:-_tmp/opy-test}
582 _python-symbols bin/oil.py oil $out_dir
583}
584
585old-style-classes() {
586 oil-python-symbols | grep -v '<'
587}
588
589# Some of these are "abstract classes" like ChildStateChange
590NotImplementedError() {
591 grep NotImplementedError */*.py
592}
593
594py-ext() {
595 # for the py-source build
596 # 35 imports
597 osh-files | xargs -- egrep 'import (fanos|libc|line_input|posix_|yajl)'
598}
599
600shell-commands() {
601 # first word on a line
602 xargs egrep --no-filename --only-matching '^[ ]*[a-z]+' |
603 sed 's/ //g' | sort | uniq -c | sort -n
604}
605
606oils-shell-commands() {
607 # common externs:
608 # 289 cat
609 # 241 mkdir
610 # 174 git
611 # 142 ls
612 # 92 sudo
613 # 86 rm
614 # 84 find
615 # 67 grep
616 # 50 wc
617 ls */*.sh | grep -v 'spec/' | shell-commands
618}
619
620other-shell-commands() {
621 # 978 rm
622 ls benchmarks/testdata/* | shell-commands
623}
624
625if test $(basename $0) = 'source-code.sh'; then
626 "$@"
627fi