OILS / soil / web.py View on Github | oilshell.org

811 lines, 265 significant
1#!/usr/bin/env python2
2"""
3soil/web.py - Dashboard that uses the "Event Sourcing" Paradigm
4
5Given state like this:
6
7https://test.oils-for-unix.org/
8 github-jobs/
9 1234/ # $GITHUB_RUN_NUMBER
10 cpp-small.tsv # benchmarks/time.py output. Success/failure for each task.
11 cpp-small.json # metadata when job is DONE
12
13 (cpp-small.wwz is linked to, but not part of the state.)
14
15 (cpp-small.state # maybe for more transient events)
16
17This script generates:
18
19https://test.oils-for-unix.org/
20 github-jobs/
21 tmp-$$.index.html # jobs for all runs
22 1234/
23 tmp-$$.index.html # jobs and tasks for a given run
24 tmp-$$.remove.txt # TODO: consolidate 'cleanup', to make it faster
25
26 # For sourcehut
27 git-0101abab/
28 tmp-$$.index.html
29
30How to test changes to this file:
31
32 $ soil/web-init.sh deploy-code
33 $ soil/web-worker.sh remote-rewrite-jobs-index github- ${GITHUB_RUN_NUMBER}
34 $ soil/web-worker.sh remote-rewrite-jobs-index srht- git-${commit_hash}
35
36"""
37from __future__ import print_function
38
39import collections
40import csv
41import datetime
42import json
43import itertools
44import os
45import re
46import sys
47from doctools import html_head
48from vendor import jsontemplate
49
50
51def log(msg, *args):
52 if args:
53 msg = msg % args
54 print(msg, file=sys.stderr)
55
56
57# *** UNUSED because it only makes sense on a dynamic web page! ***
58# Loosely based on
59# https://stackoverflow.com/questions/1551382/user-friendly-time-format-in-python
60
61SECS_IN_DAY = 86400
62
63
64def PrettyTime(now, start_time):
65 """
66 Return a pretty string like 'an hour ago', 'Yesterday', '3 months ago', 'just
67 now', etc
68 """
69 delta = now - start_time
70
71 if delta < 10:
72 return "just now"
73 if delta < 60:
74 return "%d seconds ago" % delta
75 if delta < 120:
76 return "a minute ago"
77 if delta < 3600:
78 return "%d minutes ago" % (delta // 60)
79 if delta < 7200:
80 return "an hour ago"
81 if delta < SECS_IN_DAY:
82 return "%d hours ago" % (delta // 3600)
83
84 if delta < 2 * SECS_IN_DAY:
85 return "Yesterday"
86 if delta < 7 * SECS_IN_DAY:
87 return "%d days ago" % (delta // SECS_IN_DAY)
88
89 if day_diff < 31 * SECS_IN_DAY:
90 return "%d weeks ago" % (delta / SECS_IN_DAY / 7)
91
92 if day_diff < 365:
93 return "%d months ago" % (delta / SECS_IN_DAY / 30)
94
95 return "%d years ago" % (delta / SECS_IN_DAY / 365)
96
97
98def _MinutesSeconds(num_seconds):
99 num_seconds = round(num_seconds) # round to integer
100 minutes = num_seconds / 60
101 seconds = num_seconds % 60
102 return '%d:%02d' % (minutes, seconds)
103
104
105LINE_RE = re.compile(r'(\w+)[ ]+([\d.]+)')
106
107def _ParsePullTime(time_p_str):
108 """
109 Given time -p output like
110
111 real 0.01
112 user 0.02
113 sys 0.02
114
115 Return the real time as a string, or - if we don't know it.
116 """
117 for line in time_p_str.splitlines():
118 m = LINE_RE.match(line)
119 if m:
120 name, value = m.groups()
121 if name == 'real':
122 return _MinutesSeconds(float(value))
123
124 return '-' # Not found
125
126
127DETAILS_RUN_T = jsontemplate.Template('''\
128
129<table>
130<tr class="spacer">
131 <td></td>
132</tr>
133
134<tr class="commit-row">
135 <td>
136 <code>
137 {.section github-commit-link}
138 <a href="https://github.com/oilshell/oil/commit/{commit-hash}">{commit-hash-short}</a>
139 {.end}
140
141 {.section sourcehut-commit-link}
142 <a href="https://git.sr.ht/~andyc/oil/commit/{commit-hash}">{commit-hash-short}</a>
143 {.end}
144 </code>
145 </td>
146
147 <td class="commit-line">
148 {.section github-pr}
149 <i>
150 PR <a href="https://github.com/oilshell/oil/pull/{pr-number}">#{pr-number}</a>
151 from <a href="https://github.com/oilshell/oil/tree/{head-ref}">{head-ref}</a>
152 </i>
153 {.end}
154 {.section commit-desc}
155 {@|html}
156 {.end}
157
158 {.section git-branch}
159 <br/>
160 <div style="text-align: right; font-family: monospace">{@}</div>
161 {.end}
162 </td>
163
164</tr>
165<tr class="spacer">
166 <td><td/>
167</tr>
168
169</table>
170''')
171
172
173DETAILS_TABLE_T = jsontemplate.Template('''\
174<table class="col1-right col3-right col4-right col5-right col6-right">
175
176 <thead>
177 <tr>
178 <td>ID</td>
179 <td>Job Name</td>
180 <td>Start Time</td>
181 <td>Pull Time</td>
182 <td>Run Time</td>
183 <td>Status</td>
184 </tr>
185 </thead>
186
187 {.repeated section jobs}
188 <tr>
189
190 <td>{job_num}</td>
191
192 <!-- internal link -->
193 <td> <code><a href="#job-{job-name}">{job-name}</a></code> </td>
194
195 <td><a href="{job_url}">{start_time_str}</a></td>
196 <td>
197 {.section pull_time_str}
198 <a href="{run_wwz_path}/_tmp/soil/image.html">{@}</a>
199 {.or}
200 -
201 {.end}
202 </td>
203
204 <td>{run_time_str}</td>
205
206 <td> <!-- status -->
207 {.section passed}
208 <span class="pass">pass</span>
209 {.end}
210
211 {.section failed}
212 <span class="fail">FAIL</span><br/>
213 <span class="fail-detail">
214 {.section one-failure}
215 task <code>{@}</code>
216 {.end}
217
218 {.section multiple-failures}
219 {num-failures} of {num-tasks} tasks
220 {.end}
221 </span>
222 {.end}
223 </td>
224
225 </tr>
226 {.end}
227
228</table>
229''')
230
231
232def ParseJobs(stdin):
233 """
234 Given the output of list-json, open JSON and corresponding TSV, and yield a
235 list of JSON template rows.
236 """
237 for i, line in enumerate(stdin):
238 json_path = line.strip()
239
240 #if i % 20 == 0:
241 # log('job %d = %s', i, json_path)
242
243 with open(json_path) as f:
244 meta = json.load(f)
245 #print(meta)
246
247 tsv_path = json_path[:-5] + '.tsv'
248 #log('%s', tsv_path)
249
250 all_tasks = []
251 failed_tasks = []
252 total_elapsed = 0.0
253
254 with open(tsv_path) as f:
255 reader = csv.reader(f, delimiter='\t')
256
257 try:
258 for row in reader:
259 t = {}
260 # Unpack, matching _tmp/soil/INDEX.tsv
261 ( status, elapsed,
262 t['name'], t['script_name'], t['func'], results_url) = row
263
264 t['results_url'] = None if results_url == '-' else results_url
265
266 status = int(status)
267 elapsed = float(elapsed)
268
269 t['elapsed_str'] = _MinutesSeconds(elapsed)
270
271 all_tasks.append(t)
272
273 t['status'] = status
274 if status == 0:
275 t['passed'] = True
276 else:
277 t['failed'] = True
278 failed_tasks.append(t)
279
280 total_elapsed += elapsed
281
282 except (IndexError, ValueError) as e:
283 raise RuntimeError('Error in %r: %s (%r)' % (tsv_path, e, row))
284
285 # So we can print task tables
286 meta['tasks'] = all_tasks
287
288 num_failures = len(failed_tasks)
289
290 if num_failures == 0:
291 meta['passed'] = True
292 else:
293 failed = {}
294 if num_failures == 1:
295 failed['one-failure'] = failed_tasks[0]['name']
296 else:
297 failed['multiple-failures'] = {
298 'num-failures': num_failures,
299 'num-tasks': len(all_tasks),
300 }
301 meta['failed'] = failed
302
303 meta['run_time_str'] = _MinutesSeconds(total_elapsed)
304
305 pull_time = meta.get('image-pull-time')
306 if pull_time is not None:
307 meta['pull_time_str'] = _ParsePullTime(pull_time)
308
309 start_time = meta.get('task-run-start-time')
310 if start_time is None:
311 start_time_str = '?'
312 else:
313 # Note: this is different clock! Could be desynchronized.
314 # Doesn't make sense this is static!
315 #now = time.time()
316 start_time = int(start_time)
317
318 t = datetime.datetime.fromtimestamp(start_time)
319 # %-I avoids leading 0, and is 12 hour date.
320 # lower() for 'pm' instead of 'PM'.
321 start_time_str = t.strftime('%-m/%d at %-I:%M%p').lower()
322
323 #start_time_str = PrettyTime(now, start_time)
324
325 meta['start_time_str'] = start_time_str
326
327 # Metadata for a "run". A run is for a single commit, and consists of many
328 # jobs.
329
330 meta['git-branch'] = meta.get('GITHUB_REF')
331
332 # Show the branch ref/heads/soil-staging or ref/pull/1577/merge (linkified)
333 pr_head_ref = meta.get('GITHUB_PR_HEAD_REF')
334 pr_number = meta.get('GITHUB_PR_NUMBER')
335
336 if pr_head_ref and pr_number:
337 meta['github-pr'] = {
338 'head-ref': pr_head_ref,
339 'pr-number': pr_number,
340 }
341
342 # Show the user's commit, not the merge commit
343 commit_hash = meta.get('GITHUB_PR_HEAD_SHA') or '?'
344
345 else:
346 # From soil/worker.sh save-metadata. This is intended to be
347 # CI-independent, while the environment variables above are from Github.
348 meta['commit-desc'] = meta.get('commit-line', '?')
349 commit_hash = meta.get('commit-hash') or '?'
350
351 commit_link = {
352 'commit-hash': commit_hash,
353 'commit-hash-short': commit_hash[:8],
354 }
355
356 meta['job-name'] = meta.get('job-name') or '?'
357
358 # Metadata for "Job"
359
360 # GITHUB_RUN_NUMBER (project-scoped) is shorter than GITHUB_RUN_ID (global
361 # scope)
362 github_run = meta.get('GITHUB_RUN_NUMBER')
363
364 if github_run:
365 meta['job_num'] = github_run
366 meta['index_run_url'] = '%s/' % github_run
367
368 meta['github-commit-link'] = commit_link
369
370 run_url_prefix = ''
371 else:
372 sourcehut_job_id = meta['JOB_ID']
373 meta['job_num'] = sourcehut_job_id
374 meta['index_run_url'] = 'git-%s/' % meta['commit-hash']
375
376 meta['sourcehut-commit-link'] = commit_link
377
378 # sourcehut doesn't have RUN ID, so we're in
379 # srht-jobs/git-ab01cd/index.html, and need to find srht-jobs/123/foo.wwz
380 run_url_prefix = '../%s/' % sourcehut_job_id
381
382 # For Github, we construct $JOB_URL in soil/github-actions.sh
383 meta['job_url'] = meta.get('JOB_URL') or '?'
384
385 prefix, _ = os.path.splitext(json_path) # x/y/123/myjob
386 parts = prefix.split('/')
387
388 # Paths relative to github-jobs/1234/
389 meta['run_wwz_path'] = run_url_prefix + parts[-1] + '.wwz' # myjob.wwz
390 meta['run_tsv_path'] = run_url_prefix + parts[-1] + '.tsv' # myjob.tsv
391 meta['run_json_path'] = run_url_prefix + parts[-1] + '.json' # myjob.json
392
393 # Relative to github-jobs/
394 last_two_parts = parts[-2:] # ['123', 'myjob']
395 meta['index_wwz_path'] = '/'.join(last_two_parts) + '.wwz' # 123/myjob.wwz
396
397 yield meta
398
399
400HTML_BODY_TOP_T = jsontemplate.Template('''
401 <body class="width50">
402 <p id="home-link">
403 <a href="..">Up</a>
404 | <a href="/">travis-ci.oilshell.org</a>
405 | <a href="//oilshell.org/">oilshell.org</a>
406 </p>
407
408 <h1>{title|html}</h1>
409''')
410
411HTML_BODY_BOTTOM = '''\
412 </body>
413</html>
414'''
415
416INDEX_HEADER = '''\
417<table>
418 <thead>
419 <tr>
420 <td colspan=1> Commit </td>
421 <td colspan=1> Description </td>
422 </tr>
423 </thead>
424'''
425
426INDEX_RUN_ROW_T = jsontemplate.Template('''\
427<tr class="spacer">
428 <td colspan=2></td>
429</tr>
430
431<tr class="commit-row">
432 <td>
433 <code>
434 {.section github-commit-link}
435 <a href="https://github.com/oilshell/oil/commit/{commit-hash}">{commit-hash-short}</a>
436 {.end}
437
438 {.section sourcehut-commit-link}
439 <a href="https://git.sr.ht/~andyc/oil/commit/{commit-hash}">{commit-hash-short}</a>
440 {.end}
441 </code>
442
443 </td>
444 </td>
445
446 <td class="commit-line">
447 {.section github-pr}
448 <i>
449 PR <a href="https://github.com/oilshell/oil/pull/{pr-number}">#{pr-number}</a>
450 from <a href="https://github.com/oilshell/oil/tree/{head-ref}">{head-ref}</a>
451 </i>
452 {.end}
453 {.section commit-desc}
454 {@|html}
455 {.end}
456
457 {.section git-branch}
458 <br/>
459 <div style="text-align: right; font-family: monospace">{@}</div>
460 {.end}
461 </td>
462
463</tr>
464<tr class="spacer">
465 <td colspan=2><td/>
466</tr>
467''')
468
469INDEX_JOBS_T = jsontemplate.Template('''\
470<tr>
471 <td>
472 </td>
473 <td>
474 <a href="{index_run_url}">All Jobs and Tasks</a>
475 </td>
476</tr>
477
478{.section jobs-passed}
479 <tr>
480 <td class="pass">
481 Passed
482 </td>
483 <td>
484 {.repeated section @}
485 <code class="pass">{job-name}</code>
486 <!--
487 <span class="pass"> &#x2713; </span>
488 -->
489 {.alternates with}
490 &nbsp; &nbsp;
491 {.end}
492 </td>
493 </tr>
494{.end}
495
496{.section jobs-failed}
497 <tr>
498 <td class="fail">
499 Failed
500 </td>
501 <td>
502 {.repeated section @}
503 <span class="fail"> &#x2717; </span>
504 <code><a href="{index_run_url}#job-{job-name}">{job-name}</a></code>
505
506 <span class="fail-detail">
507 {.section failed}
508 {.section one-failure}
509 - task <code>{@}</code>
510 {.end}
511
512 {.section multiple-failures}
513 - {num-failures} of {num-tasks} tasks
514 {.end}
515 {.end}
516 </span>
517
518 {.alternates with}
519 <br />
520 {.end}
521 </td>
522 </tr>
523{.end}
524
525<tr class="spacer">
526 <td colspan=3> &nbsp; </td>
527</tr>
528
529''')
530
531def PrintIndexHtml(title, groups, f=sys.stdout):
532 # Bust cache (e.g. Safari iPad seems to cache aggressively and doesn't
533 # have Ctrl-F5)
534 html_head.Write(f, title,
535 css_urls=['../web/base.css?cache=0', '../web/soil.css?cache=0'])
536
537 d = {'title': title}
538 print(HTML_BODY_TOP_T.expand(d), file=f)
539
540 print(INDEX_HEADER, file=f)
541
542 for key, jobs in groups.iteritems():
543 # All jobs have run-level metadata, so just use the first
544
545 print(INDEX_RUN_ROW_T.expand(jobs[0]), file=f)
546
547 summary = {
548 'jobs-passed': [],
549 'jobs-failed': [],
550 'index_run_url': jobs[0]['index_run_url'],
551 }
552
553 for job in jobs:
554 if job.get('passed'):
555 summary['jobs-passed'].append(job)
556 else:
557 summary['jobs-failed'].append(job)
558
559 print(INDEX_JOBS_T.expand(summary), file=f)
560
561 print(' </table>', file=f)
562 print(HTML_BODY_BOTTOM, file=f)
563
564
565TASK_TABLE_T = jsontemplate.Template('''\
566
567<h2>All Tasks</h2>
568
569<!-- right justify elapsed and status -->
570<table class="col2-right col3-right col4-right">
571
572{.repeated section jobs}
573
574<tr> <!-- link here -->
575 <td colspan=4>
576 <a name="job-{job-name}"></a>
577 </td>
578</tr>
579
580<tr style="background-color: #EEE">
581 <td colspan=3>
582 <b>{job-name}</b>
583 &nbsp;
584 &nbsp;
585 &nbsp;
586 <a href="{run_wwz_path}/">wwz</a>
587 &nbsp;
588 <a href="{run_tsv_path}">TSV</a>
589 &nbsp;
590 <a href="{run_json_path}">JSON</a>
591 <td>
592 <a href="">Up</a>
593 </td>
594</tr>
595
596<tr class="spacer">
597 <td colspan=4> &nbsp; </td>
598</tr>
599
600<tr style="font-weight: bold">
601 <td>Task</td>
602 <td>Results</td>
603 <td>Elapsed</td>
604 <td>Status</td>
605</tr>
606
607 {.repeated section tasks}
608 <tr>
609 <td>
610 <a href="{run_wwz_path}/_tmp/soil/logs/{name}.txt">{name}</a> <br/>
611 <code>{script_name} {func}</code>
612 </td>
613
614 <td>
615 {.section results_url}
616 <a href="{run_wwz_path}/{@}">Results</a>
617 {.or}
618 {.end}
619 </td>
620
621 <td>{elapsed_str}</td>
622
623 {.section passed}
624 <td>{status}</td>
625 {.end}
626 {.section failed}
627 <td class="fail">status: {status}</td>
628 {.end}
629
630 </tr>
631 {.end}
632
633<tr class="spacer">
634 <td colspan=4> &nbsp; </td>
635</tr>
636
637{.end}
638
639</table>
640
641''')
642
643
644def PrintRunHtml(title, jobs, f=sys.stdout):
645 """Print index for jobs in a single run."""
646
647 # Have to descend an extra level
648 html_head.Write(f, title,
649 css_urls=['../../web/base.css?cache=0', '../../web/soil.css?cache=0'])
650
651 d = {'title': title}
652 print(HTML_BODY_TOP_T.expand(d), file=f)
653
654 print(DETAILS_RUN_T.expand(jobs[0]), file=f)
655
656 d2 = {'jobs': jobs}
657 print(DETAILS_TABLE_T.expand(d2), file=f)
658
659 print(TASK_TABLE_T.expand(d2), file=f)
660
661 print(HTML_BODY_BOTTOM, file=f)
662
663
664def GroupJobs(jobs, key_func):
665 """
666 Expands groupby result into a simple dict
667 """
668 groups = itertools.groupby(jobs, key=key_func)
669
670 d = collections.OrderedDict()
671
672 for key, job_iter in groups:
673 jobs = list(job_iter)
674
675 jobs.sort(key=ByTaskRunStartTime, reverse=True)
676
677 d[key] = jobs
678
679 return d
680
681
682def ByTaskRunStartTime(row):
683 return int(row.get('task-run-start-time', 0))
684
685def ByCommitDate(row):
686 # Written in the shell script
687 # This is in ISO 8601 format (git log %aI), so we can sort by it.
688 return row.get('commit-date', '?')
689
690def ByCommitHash(row):
691 return row.get('commit-hash', '?')
692
693def ByGithubRun(row):
694 # Written in the shell script
695 # This is in ISO 8601 format (git log %aI), so we can sort by it.
696 return int(row.get('GITHUB_RUN_NUMBER', 0))
697
698
699def main(argv):
700 action = argv[1]
701
702 if action == 'srht-index':
703 index_out = argv[2]
704 run_index_out = argv[3]
705 run_id = argv[4] # looks like git-0101abab
706
707 assert run_id.startswith('git-'), run_id
708 commit_hash = run_id[4:]
709
710 jobs = list(ParseJobs(sys.stdin))
711
712 # sourcehut doesn't have a build number.
713 # - Sort by descnding commit date. (Minor problem: Committing on a VM with
714 # bad clock can cause commits "in the past")
715 # - Group by commit HASH, because 'git rebase' can crate different commits
716 # with the same date.
717 jobs.sort(key=ByCommitDate, reverse=True)
718 groups = GroupJobs(jobs, ByCommitHash)
719
720 title = 'Recent Jobs (sourcehut)'
721 with open(index_out, 'w') as f:
722 PrintIndexHtml(title, groups, f=f)
723
724 jobs = groups[commit_hash]
725 title = 'Jobs for commit %s' % commit_hash
726 with open(run_index_out, 'w') as f:
727 PrintRunHtml(title, jobs, f=f)
728
729 elif action == 'github-index':
730
731 index_out = argv[2]
732 run_index_out = argv[3]
733 run_id = int(argv[4]) # compared as an integer
734
735 jobs = list(ParseJobs(sys.stdin))
736
737 jobs.sort(key=ByGithubRun, reverse=True) # ordered
738 groups = GroupJobs(jobs, ByGithubRun)
739
740 title = 'Recent Jobs (Github Actions)'
741 with open(index_out, 'w') as f:
742 PrintIndexHtml(title, groups, f=f)
743
744 jobs = groups[run_id]
745 title = 'Jobs for run %d' % run_id
746
747 with open(run_index_out, 'w') as f:
748 PrintRunHtml(title, jobs, f=f)
749
750 elif action == 'cleanup':
751 try:
752 num_to_keep = int(argv[2])
753 except IndexError:
754 num_to_keep = 200
755
756 prefixes = []
757 for line in sys.stdin:
758 json_path = line.strip()
759
760 #log('%s', json_path)
761 prefixes.append(json_path[:-5])
762
763 log('%s cleanup: keep %d', sys.argv[0], num_to_keep)
764 log('%s cleanup: got %d JSON paths', sys.argv[0], len(prefixes))
765
766 # TODO: clean up git-$hash dirs
767 #
768 # github-jobs/
769 # $GITHUB_RUN_NUMBER/
770 # cpp-tarball.{json,wwz,tsv}
771 # dummy.{json,wwz,tsv}
772 # git-$hash/
773 # oils-for-unix.tar
774 #
775 # srht-jobs/
776 # 1234/
777 # cpp-tarball.{json,wwz,tsv}
778 # 1235/
779 # dummy.{json,wwz,tsv}
780 # git-$hash/
781 # index.html # HTML for this job
782 # oils-for-unix.tar
783 #
784 # We might have to read the most recent JSON, find the corresponding $hash,
785 # and print that dir.
786 #
787 # Another option is to use a real database, rather than the file system!
788
789 # Sort by 999 here
790 # travis-ci.oilshell.org/github-jobs/999/foo.json
791
792 prefixes.sort(key = lambda path: int(path.split('/')[-2]))
793
794 prefixes = prefixes[:-num_to_keep]
795
796 # Show what to delete. Then the user can pipe to xargs rm to remove it.
797 for prefix in prefixes:
798 print(prefix + '.json')
799 print(prefix + '.tsv')
800 print(prefix + '.wwz')
801
802 else:
803 raise RuntimeError('Invalid action %r' % action)
804
805
806if __name__ == '__main__':
807 try:
808 main(sys.argv)
809 except RuntimeError as e:
810 print('FATAL: %s' % e, file=sys.stderr)
811 sys.exit(1)