OILS / doctools / src_tree.py View on Github | oils.pub

474 lines, 238 significant
1#!/usr/bin/env python2
2"""src_tree.py: Publish a directory tree as HTML.
3
4TODO:
5
6- dir listing:
7 - should have columns
8 - or add line counts, and file counts?
9 - render README.md - would be nice
10
11- Could use JSON Template {.template} like test/wild_report.py
12 - for consistent header and all that
13
14AUTO
15
16- overview.html and for-translation.html should link to these files, not Github
17"""
18from __future__ import print_function
19
20import json
21import os
22import shutil
23import sys
24
25from vendor.typing import IO
26
27from doctools.util import log
28from doctools import html_head
29from test import wild_report
30from vendor import jsontemplate
31
32T = jsontemplate.Template
33
34
35def DetectType(path):
36
37 # Most support moved to src-tree.sh and micro-syntax
38
39 if path.endswith('.test.sh'):
40 return 'spec'
41
42 else:
43 return 'other'
44
45
46def Breadcrumb(rel_path, out_f, is_file=False):
47 offset = -1 if is_file else 0
48 data = wild_report.MakeNav(rel_path, root_name='OILS', offset=offset)
49 out_f.write(wild_report.NAV_TEMPLATE.expand({'nav': data}))
50
51
52# CSS class .line has white-space: pre
53
54# To avoid copy-paste problem, you could try the <div> solutions like this:
55# https://gitlab.com/gitlab-examples/python-getting-started/-/blob/master/manage.py?ref_type=heads
56
57# Note: we are compressing some stuff
58
59ROW_T = T("""\
60<tr>
61 <td class=num>{line_num}</td>
62 <td id=L{line_num}>
63 <span class="line {.section line_class}{@}{.end}">{line}</span>
64 </td>
65</tr>
66""",
67 default_formatter='html')
68
69LISTING_T = T("""\
70{.section dirs}
71<h1>Dirs</h1>
72<div id="dirs" class="listing">
73 {.repeated section @}
74 <a href="{name|htmltag}/index.html">{name|html}/</a> <br/>
75 {.end}
76</div>
77{.end}
78
79{.section files}
80<h1>Files</h1>
81<div id="files" class="listing">
82 {.repeated section @}
83 <a href="{url|htmltag}">{anchor|html}</a> <br/>
84 {.end}
85</div>
86{.end}
87
88</body>
89""")
90
91FILE_COUNTS_T = T("""\
92<div id="file-counts"> {num_lines} lines, {num_sig_lines} significant </div>
93""",
94 default_formatter='html')
95
96
97def SpecFiles(pairs, attrs_f):
98
99 for i, (path, html_out) in enumerate(pairs):
100 #log(path)
101
102 try:
103 os.makedirs(os.path.dirname(html_out))
104 except OSError:
105 pass
106
107 with open(path) as in_f, open(html_out, 'w') as out_f:
108 title = path
109
110 # How deep are we?
111 n = path.count('/') + 2
112 base_dir = '/'.join(['..'] * n)
113
114 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
115 css_urls = ['%s/web/src-tree.css' % base_dir]
116
117 html_head.Write(out_f, title, css_urls=css_urls)
118
119 out_f.write('''
120 <body class="">
121 <div id="home-link">
122 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
123 |
124 <a href="/">oils.pub</a>
125 </div>
126 <table>
127 ''' % path)
128
129 file_type = DetectType(path)
130
131 line_num = 1 # 1-based
132 for line in in_f:
133 if line.endswith('\n'):
134 line = line[:-1]
135
136 # Write line numbers
137 row = {'line_num': line_num, 'line': line}
138
139 s = line.lstrip()
140
141 if file_type == 'spec':
142 if s.startswith('####'):
143 row['line_class'] = 'spec-comment'
144 elif s.startswith('#'):
145 row['line_class'] = 'comm'
146
147 out_f.write(ROW_T.expand(row))
148
149 line_num += 1
150
151 # could be parsed by 'dirs'
152 print('%s lines=%d' % (path, line_num), file=attrs_f)
153
154 out_f.write('''
155 </table>
156 </body>
157 </html>''')
158
159 return i + 1
160
161
162def ReadFragments(in_f):
163 while True:
164 path = ReadNetString(in_f)
165 if path is None:
166 break
167
168 html_frag = ReadNetString(in_f)
169 if html_frag is None:
170 raise RuntimeError('Expected 2nd record (HTML fragment)')
171
172 s = ReadNetString(in_f)
173 if s is None:
174 raise RuntimeError('Expected 3rd record (file summary)')
175
176 summary = json.loads(s)
177
178 yield path, html_frag, summary
179
180
181def WriteHtmlFragments(in_f, out_dir, attrs_f=sys.stdout):
182
183 i = 0
184 for rel_path, html_frag, summary in ReadFragments(in_f):
185 html_size = len(html_frag)
186 if html_size > 300000:
187 out_path = os.path.join(out_dir, rel_path)
188 try:
189 os.makedirs(os.path.dirname(out_path))
190 except OSError:
191 pass
192
193 shutil.copyfile(rel_path, out_path)
194
195 # Attrs are parsed by MakeTree(), and then used by WriteDirsHtml().
196 # So we can print the right link.
197 print('%s raw=1' % rel_path, file=attrs_f)
198
199 file_size = os.path.getsize(rel_path)
200 log('Big HTML fragment of %.1f KB', float(html_size) / 1000)
201 log('Copied %s -> %s, %.1f KB', rel_path, out_path,
202 float(file_size) / 1000)
203
204 continue
205
206 html_out = os.path.join(out_dir, rel_path + '.html')
207
208 try:
209 os.makedirs(os.path.dirname(html_out))
210 except OSError:
211 pass
212
213 with open(html_out, 'w') as out_f:
214 title = rel_path
215
216 # How deep are we?
217 n = rel_path.count('/') + 2
218 base_dir = '/'.join(['..'] * n)
219
220 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
221 css_urls = ['%s/web/src-tree.css' % base_dir]
222 html_head.Write(out_f, title, css_urls=css_urls)
223
224 out_f.write('''
225 <body class="">
226 <p>
227 ''')
228 Breadcrumb(rel_path, out_f, is_file=True)
229
230 out_f.write('''
231 <span id="home-link">
232 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
233 |
234 <a href="/">oils.pub</a>
235 </span>
236 </p>
237 ''' % rel_path)
238
239 out_f.write(FILE_COUNTS_T.expand(summary))
240
241 out_f.write('<table>')
242 out_f.write(html_frag)
243
244 print('%s lines=%d' % (rel_path, summary['num_lines']),
245 file=attrs_f)
246
247 out_f.write('''
248 </table>
249 </body>
250 </html>''')
251
252 i += 1
253
254 log('Wrote %d HTML fragments', i)
255
256
257class DirNode:
258 """Entry in the file system tree.
259
260 Similar to test/wild_report.py
261 """
262
263 def __init__(self):
264 # type: () -> None
265 self.files = {} # filename -> attrs dict
266 self.dirs = {} # subdir name -> DirNode object
267
268 # Can accumulate total lines here
269 self.subtree_stats = {} # name -> value
270
271
272def DebugPrint(node, indent=0):
273 """Pretty-print our tree data structure."""
274 ind = indent * ' '
275 #print('FILES', node.files.keys())
276 for name in node.files:
277 print('%s%s - %s' % (ind, name, node.files[name]))
278
279 for name, child in node.dirs.iteritems():
280 print('%s%s/ - %s' % (ind, name, child.subtree_stats))
281 DebugPrint(child, indent=indent + 1)
282
283
284def UpdateNodes(node, path_parts, attrs):
285 """Similar to test/wild_report.py."""
286
287 first = path_parts[0]
288 rest = path_parts[1:]
289
290 if rest: # update an intermediate node
291 if first in node.dirs:
292 child = node.dirs[first]
293 else:
294 child = DirNode()
295 node.dirs[first] = child
296
297 UpdateNodes(child, rest, attrs)
298 # TODO: Update subtree_stats
299
300 else:
301 # leaf node
302 node.files[first] = attrs
303
304
305def MakeTree(stdin, root_node):
306 """Reads a stream of lines Each line contains a path and key=value attrs.
307
308 - Doesn't handle filenames with spaces
309 - Doesn't handle empty dirs that are leaves (since only files are first
310 class)
311 """
312 for line in sys.stdin:
313 parts = line.split()
314 path = parts[0]
315
316 # Examples:
317 # {'lines': '345'}
318 # {'raw': '1'}
319 attrs = {}
320 for part in parts[1:]:
321 k, v = part.split('=')
322 attrs[k] = v
323
324 path_parts = path.split('/')
325 UpdateNodes(root_node, path_parts, attrs)
326
327
328def WriteDirsHtml(node, out_dir, rel_path='', base_url=''):
329 #log('WriteDirectory %s %s %s', out_dir, rel_path, base_url)
330
331 files = []
332 for name in sorted(node.files):
333 attrs = node.files[name]
334
335 # Big files are raw, e.g. match.re2c.h and syntax_asdl.py
336 url = name if attrs.get('raw') else '%s.html' % name
337 f = {'url': url, 'anchor': name}
338 files.append(f)
339
340 dirs = []
341 for name in sorted(node.dirs):
342 dirs.append({'name': name})
343
344 data = {'files': files, 'dirs': dirs}
345 body = LISTING_T.expand(data)
346
347 path = os.path.join(out_dir, 'index.html')
348 with open(path, 'w') as f:
349
350 title = '%s - Listing' % rel_path
351 prefix = '%s../..' % base_url
352 css_urls = ['%s/web/base.css' % prefix, '%s/web/src-tree.css' % prefix]
353 html_head.Write(f, title, css_urls=css_urls)
354
355 f.write('''
356 <body>
357 <p>
358 ''')
359 Breadcrumb(rel_path, f)
360
361 f.write('''
362 <span id="home-link">
363 <a href="/">oils.pub</a>
364 </span>
365 </p>
366 ''')
367
368 f.write(body)
369
370 f.write('</html>')
371
372 # Recursive
373 for name, child in node.dirs.iteritems():
374 child_out = os.path.join(out_dir, name)
375 child_rel = os.path.join(rel_path, name)
376 child_base = base_url + '../'
377 WriteDirsHtml(child,
378 child_out,
379 rel_path=child_rel,
380 base_url=child_base)
381
382
383def ReadNetString(in_f):
384 # type: (IO[str]) -> str
385
386 digits = []
387 for i in xrange(10): # up to 10 digits
388 c = in_f.read(1)
389 if c == '':
390 return None # EOF
391
392 if c == ':':
393 break
394
395 if not c.isdigit():
396 raise RuntimeError('Bad byte %r' % c)
397
398 digits.append(c)
399
400 if c != ':':
401 raise RuntimeError('Expected colon, got %r' % c)
402
403 n = int(''.join(digits))
404
405 s = in_f.read(n)
406 if len(s) != n:
407 raise RuntimeError('Expected %d bytes, got %d' % (n, len(s)))
408
409 c = in_f.read(1)
410 if c != ',':
411 raise RuntimeError('Expected comma, got %r' % c)
412
413 return s
414
415
416def main(argv):
417 action = argv[1]
418
419 if action == 'spec-files':
420 # Policy for _tmp/spec/osh-minimal/foo.test.html
421 # This just changes the HTML names?
422
423 out_dir = argv[2]
424 spec_names = argv[3:]
425
426 pairs = []
427 for name in spec_names:
428 src = 'spec/%s.test.sh' % name
429 html_out = os.path.join(out_dir, '%s.test.html' % name)
430 pairs.append((src, html_out))
431
432 attrs_f = sys.stdout
433 n = SpecFiles(pairs, attrs_f)
434 log('%s: Wrote %d HTML files -> %s', os.path.basename(sys.argv[0]), n,
435 out_dir)
436
437 elif action == 'smoosh-file':
438 # TODO: Should fold this generated code into the source tree, and run in CI
439
440 in_path = argv[2]
441 out_path = argv[3]
442 pairs = [(in_path, out_path)]
443
444 attrs_f = sys.stdout
445 n = SpecFiles(pairs, attrs_f)
446 log('%s: %s -> %s', os.path.basename(sys.argv[0]), in_path, out_path)
447
448 elif action == 'write-html-fragments':
449
450 out_dir = argv[2]
451 WriteHtmlFragments(sys.stdin, out_dir)
452
453 elif action == 'dirs':
454 # stdin: a bunch of merged ATTRs file?
455
456 # We load them, and write a whole tree?
457 out_dir = argv[2]
458
459 # I think we make a big data structure here
460
461 root_node = DirNode()
462 MakeTree(sys.stdin, root_node)
463
464 if 0:
465 DebugPrint(root_node)
466
467 WriteDirsHtml(root_node, out_dir)
468
469 else:
470 raise RuntimeError('Invalid action %r' % action)
471
472
473if __name__ == '__main__':
474 main(sys.argv)