OILS / doctools / src_tree.py View on Github | oilshell.org

462 lines, 231 significant
1#!/usr/bin/env python2
2"""
3src_tree.py: Publish a directory tree as HTML
4
5TODO:
6
7- dir listing:
8 - should have columns
9 - or add line counts, and file counts?
10 - render README.md - would be nice
11
12- Could use JSON Template {.template} like test/wild_report.py
13 - for consistent header and all that
14
15AUTO
16
17- overview.html and for-translation.html should link to these files, not Github
18"""
19from __future__ import print_function
20
21import json
22import os
23import shutil
24import sys
25
26from doctools.util import log
27from doctools import html_head
28from test import wild_report
29from vendor import jsontemplate
30
31T = jsontemplate.Template
32
33
34def DetectType(path):
35
36 # Most support moved to src-tree.sh and micro-syntax
37
38 if path.endswith('.test.sh'):
39 return 'spec'
40
41 else:
42 return 'other'
43
44
45def Breadcrumb(rel_path, out_f, is_file=False):
46 offset = -1 if is_file else 0
47 data = wild_report.MakeNav(rel_path, root_name='OILS', offset=offset)
48 out_f.write(wild_report.NAV_TEMPLATE.expand({'nav': data}))
49
50
51# CSS class .line has white-space: pre
52
53# To avoid copy-paste problem, you could try the <div> solutions like this:
54# https://gitlab.com/gitlab-examples/python-getting-started/-/blob/master/manage.py?ref_type=heads
55
56# Note: we are compressing some stuff
57
58ROW_T = T("""\
59<tr>
60 <td class=num>{line_num}</td>
61 <td id=L{line_num}>
62 <span class="line {.section line_class}{@}{.end}">{line}</span>
63 </td>
64</tr>
65""", default_formatter='html')
66
67
68LISTING_T = T("""\
69{.section dirs}
70<h1>Dirs</h1>
71<div id="dirs" class="listing">
72 {.repeated section @}
73 <a href="{name|htmltag}/index.html">{name|html}/</a> <br/>
74 {.end}
75</div>
76{.end}
77
78{.section files}
79<h1>Files</h1>
80<div id="files" class="listing">
81 {.repeated section @}
82 <a href="{url|htmltag}">{anchor|html}</a> <br/>
83 {.end}
84</div>
85{.end}
86
87</body>
88""")
89
90FILE_COUNTS_T = T("""\
91<div id="file-counts"> {num_lines} lines, {num_sig_lines} significant </div>
92""", default_formatter='html')
93
94
95def SpecFiles(pairs, attrs_f):
96
97 for i, (path, html_out) in enumerate(pairs):
98 #log(path)
99
100 try:
101 os.makedirs(os.path.dirname(html_out))
102 except OSError:
103 pass
104
105 with open(path) as in_f, open(html_out, 'w') as out_f:
106 title = path
107
108 # How deep are we?
109 n = path.count('/') + 2
110 base_dir = '/'.join(['..'] * n)
111
112 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
113 css_urls = ['%s/web/src-tree.css' % base_dir]
114
115 html_head.Write(out_f, title, css_urls=css_urls)
116
117 out_f.write('''
118 <body class="">
119 <div id="home-link">
120 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
121 |
122 <a href="/">oilshell.org</a>
123 </div>
124 <table>
125 ''' % path)
126
127 file_type = DetectType(path)
128
129 line_num = 1 # 1-based
130 for line in in_f:
131 if line.endswith('\n'):
132 line = line[:-1]
133
134 # Write line numbers
135 row = {'line_num': line_num, 'line': line}
136
137 s = line.lstrip()
138
139 if file_type == 'spec':
140 if s.startswith('####'):
141 row['line_class'] = 'spec-comment'
142 elif s.startswith('#'):
143 row['line_class'] = 'comm'
144
145 out_f.write(ROW_T.expand(row))
146
147 line_num += 1
148
149 # could be parsed by 'dirs'
150 print('%s lines=%d' % (path, line_num), file=attrs_f)
151
152 out_f.write('''
153 </table>
154 </body>
155 </html>''')
156
157 return i + 1
158
159
160def ReadFragments(in_f):
161 while True:
162 path = ReadNetString(in_f)
163 if path is None:
164 break
165
166 html_frag = ReadNetString(in_f)
167 if html_frag is None:
168 raise RuntimeError('Expected 2nd record (HTML fragment)')
169
170 s = ReadNetString(in_f)
171 if s is None:
172 raise RuntimeError('Expected 3rd record (file summary)')
173
174 summary = json.loads(s)
175
176 yield path, html_frag, summary
177
178
179def WriteHtmlFragments(in_f, out_dir, attrs_f=sys.stdout):
180
181 i = 0
182 for rel_path, html_frag, summary in ReadFragments(in_f):
183 html_size = len(html_frag)
184 if html_size > 300000:
185 out_path = os.path.join(out_dir, rel_path)
186 try:
187 os.makedirs(os.path.dirname(out_path))
188 except OSError:
189 pass
190
191 shutil.copyfile(rel_path, out_path)
192
193 # Attrs are parsed by MakeTree(), and then used by WriteDirsHtml().
194 # So we can print the right link.
195 print('%s raw=1' % rel_path, file=attrs_f)
196
197 file_size = os.path.getsize(rel_path)
198 log('Big HTML fragment of %.1f KB', float(html_size) / 1000)
199 log('Copied %s -> %s, %.1f KB', rel_path, out_path, float(file_size) / 1000)
200
201 continue
202
203 html_out = os.path.join(out_dir, rel_path + '.html')
204
205 try:
206 os.makedirs(os.path.dirname(html_out))
207 except OSError:
208 pass
209
210 with open(html_out, 'w') as out_f:
211 title = rel_path
212
213 # How deep are we?
214 n = rel_path.count('/') + 2
215 base_dir = '/'.join(['..'] * n)
216
217 #css_urls = ['%s/web/base.css' % base_dir, '%s/web/src-tree.css' % base_dir]
218 css_urls = ['%s/web/src-tree.css' % base_dir]
219 html_head.Write(out_f, title, css_urls=css_urls)
220
221 out_f.write('''
222 <body class="">
223 <p>
224 ''')
225 Breadcrumb(rel_path, out_f, is_file=True)
226
227 out_f.write('''
228 <span id="home-link">
229 <a href="https://github.com/oilshell/oil/blob/master/%s">View on Github</a>
230 |
231 <a href="/">oilshell.org</a>
232 </span>
233 </p>
234 ''' % rel_path)
235
236 out_f.write(FILE_COUNTS_T.expand(summary))
237
238 out_f.write('<table>')
239 out_f.write(html_frag)
240
241 print('%s lines=%d' % (rel_path, summary['num_lines']), file=attrs_f)
242
243 out_f.write('''
244 </table>
245 </body>
246 </html>''')
247
248 i += 1
249
250 log('Wrote %d HTML fragments', i)
251
252
253class DirNode:
254 """Entry in the file system tree.
255
256 Similar to test/wild_report.py
257 """
258
259 def __init__(self):
260 self.files = {} # filename -> attrs dict
261 self.dirs = {} # subdir name -> DirNode object
262
263 # Can accumulate total lines here
264 self.subtree_stats = {} # name -> value
265
266
267def DebugPrint(node, indent=0):
268 """Debug print."""
269 ind = indent * ' '
270 #print('FILES', node.files.keys())
271 for name in node.files:
272 print('%s%s - %s' % (ind, name, node.files[name]))
273
274 for name, child in node.dirs.iteritems():
275 print('%s%s/ - %s' % (ind, name, child.subtree_stats))
276 DebugPrint(child, indent=indent+1)
277
278
279def UpdateNodes(node, path_parts, attrs):
280 """Similar to test/wild_report.y"""
281
282 first = path_parts[0]
283 rest = path_parts[1:]
284
285 if rest: # update an intermediate node
286 if first in node.dirs:
287 child = node.dirs[first]
288 else:
289 child = DirNode()
290 node.dirs[first] = child
291
292 UpdateNodes(child, rest, attrs)
293
294 else:
295 # leaf node
296 node.files[first] = attrs
297
298
299def MakeTree(stdin, root_node):
300 for line in sys.stdin:
301 parts = line.split()
302 path = parts[0]
303
304 # Examples:
305 # {'lines': '345'}
306 # {'raw': '1'}
307 attrs = {}
308 for part in parts[1:]:
309 k, v = part.split('=')
310 attrs[k] = v
311
312 path_parts = path.split('/')
313 UpdateNodes(root_node, path_parts, attrs)
314
315
316def WriteDirsHtml(node, out_dir, rel_path='', base_url=''):
317 #log('WriteDirectory %s %s %s', out_dir, rel_path, base_url)
318
319 files = []
320 for name in sorted(node.files):
321 attrs = node.files[name]
322
323 # Big files are raw, e.g. match.re2c.h and syntax_asdl.py
324 url = name if attrs.get('raw') else '%s.html' % name
325 f = {'url': url, 'anchor': name}
326 files.append(f)
327
328 dirs = []
329 for name in sorted(node.dirs):
330 dirs.append({'name': name})
331
332 data = {'files': files, 'dirs': dirs}
333 body = LISTING_T.expand(data)
334
335 path = os.path.join(out_dir, 'index.html')
336 with open(path, 'w') as f:
337
338 title = '%s - Listing' % rel_path
339 prefix = '%s../..' % base_url
340 css_urls = ['%s/web/base.css' % prefix, '%s/web/src-tree.css' % prefix]
341 html_head.Write(f, title, css_urls=css_urls)
342
343 f.write('''
344 <body>
345 <p>
346 ''')
347 Breadcrumb(rel_path, f)
348
349 f.write('''
350 <span id="home-link">
351 <a href="/">oilshell.org</a>
352 </span>
353 </p>
354 ''')
355
356
357 f.write(body)
358
359 f.write('</html>')
360
361 # Recursive
362 for name, child in node.dirs.iteritems():
363 child_out = os.path.join(out_dir, name)
364 child_rel = os.path.join(rel_path, name)
365 child_base = base_url + '../'
366 WriteDirsHtml(child, child_out, rel_path=child_rel,
367 base_url=child_base)
368
369
370def ReadNetString(in_f):
371
372 digits = []
373 for i in xrange(10): # up to 10 digits
374 c = in_f.read(1)
375 if c == '':
376 return None # EOF
377
378 if c == ':':
379 break
380
381 if not c.isdigit():
382 raise RuntimeError('Bad byte %r' % c)
383
384 digits.append(c)
385
386 if c != ':':
387 raise RuntimeError('Expected colon, got %r' % c)
388
389 n = int(''.join(digits))
390
391 s = in_f.read(n)
392 if len(s) != n:
393 raise RuntimeError('Expected %d bytes, got %d' % (n, len(s)))
394
395 c = in_f.read(1)
396 if c != ',':
397 raise RuntimeError('Expected comma, got %r' % c)
398
399 return s
400
401
402def main(argv):
403 action = argv[1]
404
405 if action == 'spec-files':
406 # Policy for _tmp/spec/osh-minimal/foo.test.html
407 # This just changes the HTML names?
408
409 out_dir = argv[2]
410 spec_names = argv[3:]
411
412 pairs = []
413 for name in spec_names:
414 src = 'spec/%s.test.sh' % name
415 html_out = os.path.join(out_dir, '%s.test.html' % name)
416 pairs.append((src, html_out))
417
418 attrs_f = sys.stdout
419 n = SpecFiles(pairs, attrs_f)
420 log('%s: Wrote %d HTML files -> %s', os.path.basename(sys.argv[0]), n,
421 out_dir)
422
423 elif action == 'smoosh-file':
424 # TODO: Should fold this generated code into the source tree, and run in CI
425
426 in_path = argv[2]
427 out_path = argv[3]
428 pairs = [(in_path, out_path)]
429
430 attrs_f = sys.stdout
431 n = SpecFiles(pairs, attrs_f)
432 log('%s: %s -> %s', os.path.basename(sys.argv[0]), in_path, out_path)
433
434 elif action == 'write-html-fragments':
435
436 out_dir = argv[2]
437 WriteHtmlFragments(sys.stdin, out_dir)
438
439 elif action == 'dirs':
440 # stdin: a bunch of merged ATTRs file?
441
442 # We load them, and write a whole tree?
443 out_dir = argv[2]
444
445 # I think we make a big data structure here
446
447 root_node = DirNode()
448 MakeTree(sys.stdin, root_node)
449
450 if 0:
451 DebugPrint(root_node)
452
453 WriteDirsHtml(root_node, out_dir)
454
455 else:
456 raise RuntimeError('Invalid action %r' % action)
457
458
459if __name__ == '__main__':
460 main(sys.argv)
461
462# vim: sw=2