OILS / doctools / src_tree.py View on Github | oilshell.org

471 lines, 231 significant
1#!/usr/bin/env python2
2"""
3src_tree.py: Publish a directory tree as HTML
4
5TODO:
6
7- dir listing:
8 - should have columns
9 - or add line counts, and file counts?
10 - render README.md - would be nice
11
12- Could use JSON Template {.template} like test/wild_report.py
13 - for consistent header and all that
14
15AUTO
16
17- overview.html and for-translation.html should link to these files, not Github
18"""
19from __future__ import print_function
20
21import json
22import os
23import shutil
24import sys
25
26from doctools.util import log
27from doctools import html_head
28from test import wild_report
29from vendor import jsontemplate
30
31T = jsontemplate.Template
32
33
34def DetectType(path):
35
36 # Most support moved to src-tree.sh and micro-syntax
37
38 if path.endswith('.test.sh'):
39 return 'spec'
40
41 else:
42 return 'other'
43
44
45def Breadcrumb(rel_path, out_f, is_file=False):
46 offset = -1 if is_file else 0
47 data = wild_report.MakeNav(rel_path, root_name='OILS', offset=offset)
48 out_f.write(wild_report.NAV_TEMPLATE.expand({'nav': data}))
49
50
51# CSS class .line has white-space: pre
52
53# To avoid copy-paste problem, you could try the <div> solutions like this:
54# https://gitlab.com/gitlab-examples/python-getting-started/-/blob/master/manage.py?ref_type=heads
55
56# Note: we are compressing some stuff
57
58ROW_T = T("""\
59<tr>
60 <td class=num>{line_num}</td>
61 <td id=L{line_num}>
62 <span class="line {.section line_class}{@}{.end}">{line}</span>
63 </td>
64</tr>
65""", default_formatter='html')
66
67
68LISTING_T = T("""\
69{.section dirs}
70<h1>Dirs</h1>
71<div id="dirs" class="listing">
72 {.repeated section @}
73 <a href="{name|htmltag}/index.html">{name|html}/</a> <br/>
74 {.end}
75</div>
76{.end}
77
78{.section files}
79<h1>Files</h1>
80<div id="files" class="listing">
81 {.repeated section @}
82 <a href="{url|htmltag}">{anchor|html}</a> <br/>
83 {.end}
84</div>
85{.end}
86
87</body>
88""")
89
90FILE_COUNTS_T = T("""\
91<div id="file-counts"> {num_lines} lines, {num_sig_lines} significant </div>
92""", default_formatter='html')
93
94
95def SpecFiles(pairs, attrs_f):
96
97 for i, (path, html_out) in enumerate(pairs):
98 #log(path)
99
100 try:
101 os.makedirs(os.path.dirname(html_out))
102 except OSError:
103 pass
104
105 with open(path) as in_f, open(html_out, 'w') as out_f:
106 title = path
107
108 # How deep are we?
109 n = path.count('/') + 2
110 base_dir = '/'.join(['..'] * n)
111
112 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
113 css_urls = ['%s/web/src-tree.css' % base_dir]
114
115 html_head.Write(out_f, title, css_urls=css_urls)
116
117 out_f.write('''
118 <body class="">
119 <div id="home-link">
120 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
121 |
122 <a href="/">oilshell.org</a>
123 </div>
124 <table>
125 ''' % path)
126
127 file_type = DetectType(path)
128
129 line_num = 1 # 1-based
130 for line in in_f:
131 if line.endswith('\n'):
132 line = line[:-1]
133
134 # Write line numbers
135 row = {'line_num': line_num, 'line': line}
136
137 s = line.lstrip()
138
139 if file_type == 'spec':
140 if s.startswith('####'):
141 row['line_class'] = 'spec-comment'
142 elif s.startswith('#'):
143 row['line_class'] = 'comm'
144
145 out_f.write(ROW_T.expand(row))
146
147 line_num += 1
148
149 # could be parsed by 'dirs'
150 print('%s lines=%d' % (path, line_num), file=attrs_f)
151
152 out_f.write('''
153 </table>
154 </body>
155 </html>''')
156
157 return i + 1
158
159
160def ReadFragments(in_f):
161 while True:
162 path = ReadNetString(in_f)
163 if path is None:
164 break
165
166 html_frag = ReadNetString(in_f)
167 if html_frag is None:
168 raise RuntimeError('Expected 2nd record (HTML fragment)')
169
170 s = ReadNetString(in_f)
171 if s is None:
172 raise RuntimeError('Expected 3rd record (file summary)')
173
174 summary = json.loads(s)
175
176 yield path, html_frag, summary
177
178
179def WriteHtmlFragments(in_f, out_dir, attrs_f=sys.stdout):
180
181 i = 0
182 for rel_path, html_frag, summary in ReadFragments(in_f):
183 html_size = len(html_frag)
184 if html_size > 300000:
185 out_path = os.path.join(out_dir, rel_path)
186 try:
187 os.makedirs(os.path.dirname(out_path))
188 except OSError:
189 pass
190
191 shutil.copyfile(rel_path, out_path)
192
193 # Attrs are parsed by MakeTree(), and then used by WriteDirsHtml().
194 # So we can print the right link.
195 print('%s raw=1' % rel_path, file=attrs_f)
196
197 file_size = os.path.getsize(rel_path)
198 log('Big HTML fragment of %.1f KB', float(html_size) / 1000)
199 log('Copied %s -> %s, %.1f KB', rel_path, out_path, float(file_size) / 1000)
200
201 continue
202
203 html_out = os.path.join(out_dir, rel_path + '.html')
204
205 try:
206 os.makedirs(os.path.dirname(html_out))
207 except OSError:
208 pass
209
210 with open(html_out, 'w') as out_f:
211 title = rel_path
212
213 # How deep are we?
214 n = rel_path.count('/') + 2
215 base_dir = '/'.join(['..'] * n)
216
217 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
218 css_urls = ['%s/web/src-tree.css' % base_dir]
219 html_head.Write(out_f, title, css_urls=css_urls)
220
221 out_f.write('''
222 <body class="">
223 <p>
224 ''')
225 Breadcrumb(rel_path, out_f, is_file=True)
226
227 out_f.write('''
228 <span id="home-link">
229 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
230 |
231 <a href="/">oilshell.org</a>
232 </span>
233 </p>
234 ''' % rel_path)
235
236 out_f.write(FILE_COUNTS_T.expand(summary))
237
238 out_f.write('<table>')
239 out_f.write(html_frag)
240
241 print('%s lines=%d' % (rel_path, summary['num_lines']), file=attrs_f)
242
243 out_f.write('''
244 </table>
245 </body>
246 </html>''')
247
248 i += 1
249
250 log('Wrote %d HTML fragments', i)
251
252
253class DirNode:
254 """Entry in the file system tree.
255
256 Similar to test/wild_report.py
257 """
258
259 def __init__(self):
260 self.files = {} # filename -> attrs dict
261 self.dirs = {} # subdir name -> DirNode object
262
263 # Can accumulate total lines here
264 self.subtree_stats = {} # name -> value
265
266
267def DebugPrint(node, indent=0):
268 """Debug print."""
269 ind = indent * ' '
270 #print('FILES', node.files.keys())
271 for name in node.files:
272 print('%s%s - %s' % (ind, name, node.files[name]))
273
274 for name, child in node.dirs.iteritems():
275 print('%s%s/ - %s' % (ind, name, child.subtree_stats))
276 DebugPrint(child, indent=indent+1)
277
278
279def UpdateNodes(node, path_parts, attrs):
280 """Similar to test/wild_report.py"""
281
282 first = path_parts[0]
283 rest = path_parts[1:]
284
285 if rest: # update an intermediate node
286 if first in node.dirs:
287 child = node.dirs[first]
288 else:
289 child = DirNode()
290 node.dirs[first] = child
291
292 UpdateNodes(child, rest, attrs)
293 # TODO: Update subtree_stats
294
295 else:
296 # leaf node
297 node.files[first] = attrs
298
299
300def MakeTree(stdin, root_node):
301 """
302 Reads a stream of lines
303 Each line contains a path and key=value attrs
304
305 - Doesn't handle filenames with spaces
306 - Doesn't handle empty dirs that are leaves (since only files are first
307 class)
308 """
309 for line in sys.stdin:
310 parts = line.split()
311 path = parts[0]
312
313 # Examples:
314 # {'lines': '345'}
315 # {'raw': '1'}
316 attrs = {}
317 for part in parts[1:]:
318 k, v = part.split('=')
319 attrs[k] = v
320
321 path_parts = path.split('/')
322 UpdateNodes(root_node, path_parts, attrs)
323
324
325def WriteDirsHtml(node, out_dir, rel_path='', base_url=''):
326 #log('WriteDirectory %s %s %s', out_dir, rel_path, base_url)
327
328 files = []
329 for name in sorted(node.files):
330 attrs = node.files[name]
331
332 # Big files are raw, e.g. match.re2c.h and syntax_asdl.py
333 url = name if attrs.get('raw') else '%s.html' % name
334 f = {'url': url, 'anchor': name}
335 files.append(f)
336
337 dirs = []
338 for name in sorted(node.dirs):
339 dirs.append({'name': name})
340
341 data = {'files': files, 'dirs': dirs}
342 body = LISTING_T.expand(data)
343
344 path = os.path.join(out_dir, 'index.html')
345 with open(path, 'w') as f:
346
347 title = '%s - Listing' % rel_path
348 prefix = '%s../..' % base_url
349 css_urls = ['%s/web/base.css' % prefix, '%s/web/src-tree.css' % prefix]
350 html_head.Write(f, title, css_urls=css_urls)
351
352 f.write('''
353 <body>
354 <p>
355 ''')
356 Breadcrumb(rel_path, f)
357
358 f.write('''
359 <span id="home-link">
360 <a href="/">oilshell.org</a>
361 </span>
362 </p>
363 ''')
364
365
366 f.write(body)
367
368 f.write('</html>')
369
370 # Recursive
371 for name, child in node.dirs.iteritems():
372 child_out = os.path.join(out_dir, name)
373 child_rel = os.path.join(rel_path, name)
374 child_base = base_url + '../'
375 WriteDirsHtml(child, child_out, rel_path=child_rel,
376 base_url=child_base)
377
378
379def ReadNetString(in_f):
380
381 digits = []
382 for i in xrange(10): # up to 10 digits
383 c = in_f.read(1)
384 if c == '':
385 return None # EOF
386
387 if c == ':':
388 break
389
390 if not c.isdigit():
391 raise RuntimeError('Bad byte %r' % c)
392
393 digits.append(c)
394
395 if c != ':':
396 raise RuntimeError('Expected colon, got %r' % c)
397
398 n = int(''.join(digits))
399
400 s = in_f.read(n)
401 if len(s) != n:
402 raise RuntimeError('Expected %d bytes, got %d' % (n, len(s)))
403
404 c = in_f.read(1)
405 if c != ',':
406 raise RuntimeError('Expected comma, got %r' % c)
407
408 return s
409
410
411def main(argv):
412 action = argv[1]
413
414 if action == 'spec-files':
415 # Policy for _tmp/spec/osh-minimal/foo.test.html
416 # This just changes the HTML names?
417
418 out_dir = argv[2]
419 spec_names = argv[3:]
420
421 pairs = []
422 for name in spec_names:
423 src = 'spec/%s.test.sh' % name
424 html_out = os.path.join(out_dir, '%s.test.html' % name)
425 pairs.append((src, html_out))
426
427 attrs_f = sys.stdout
428 n = SpecFiles(pairs, attrs_f)
429 log('%s: Wrote %d HTML files -> %s', os.path.basename(sys.argv[0]), n,
430 out_dir)
431
432 elif action == 'smoosh-file':
433 # TODO: Should fold this generated code into the source tree, and run in CI
434
435 in_path = argv[2]
436 out_path = argv[3]
437 pairs = [(in_path, out_path)]
438
439 attrs_f = sys.stdout
440 n = SpecFiles(pairs, attrs_f)
441 log('%s: %s -> %s', os.path.basename(sys.argv[0]), in_path, out_path)
442
443 elif action == 'write-html-fragments':
444
445 out_dir = argv[2]
446 WriteHtmlFragments(sys.stdin, out_dir)
447
448 elif action == 'dirs':
449 # stdin: a bunch of merged ATTRs file?
450
451 # We load them, and write a whole tree?
452 out_dir = argv[2]
453
454 # I think we make a big data structure here
455
456 root_node = DirNode()
457 MakeTree(sys.stdin, root_node)
458
459 if 0:
460 DebugPrint(root_node)
461
462 WriteDirsHtml(root_node, out_dir)
463
464 else:
465 raise RuntimeError('Invalid action %r' % action)
466
467
468if __name__ == '__main__':
469 main(sys.argv)
470
471# vim: sw=2