OILS / build / doc.sh View on Github | oilshell.org

682 lines, 361 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oil-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=. doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 error-handling
79 error-catalog
80 json
81 hay
82 simple-word-eval
83 quirks
84 warts
85
86 eggex
87 ysh-regex-api
88 upgrade-breakage
89 ysh-tour
90
91 style-guide
92 novelties
93
94 proc-func
95 block-literals
96
97 # Data language
98 qsn
99 qtt
100 j8-notation
101 pretty-printing
102
103 doc-toolchain
104 doc-plugins
105 idioms
106 shell-idioms
107 ysh-faq
108
109 language-influences
110 ysh-vs-python
111 ysh-vs-shell
112
113 syntactic-concepts
114 syntax-feelings
115 command-vs-expression-mode
116
117 # needs polish
118 # Note: docs about the YSH are prefixed 'ysh-'.
119 # data-model and command-vs-expression-mode span both OSH and YSH
120
121 index
122 faq-doc
123
124 options
125
126 old/index
127 old/project-tour
128 old/legacy-array
129 old/ysh-keywords
130 old/modules
131 old/expression-language
132 old/word-language
133 old/errors
134 old/ysh-builtins
135
136 io-builtins
137 unicode
138 framing
139 xtrace
140 headless
141 completion
142 strings
143 variables
144
145 # Internal stuff
146 interpreter-state
147 process-model
148 architecture-notes
149 parser-architecture
150)
151
152# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
153# locale), which is loaded by Python into say u'\u5e74'. But the default
154# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
155# fail.
156#
157# I believe --rfc-e-mail should never output a Unicode character.
158#
159# A better fix would be to implement json_utf8.load(f), which doesn't decode
160# into unicode instances. This would remove useless conversions.
161
162readonly TIMESTAMP=$(date --rfc-email)
163
164split-and-render() {
165 local src=${1:-doc/known-differences.md}
166
167 local rel_path=${src%'.md'} # doc/known-differences
168 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
169
170 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
171 local web_url=${3:-'../web'}
172
173 mkdir -v -p $(dirname $out) $tmp_prefix
174
175 # Also add could add css_files. The one in the file takes precedence always?
176
177 # css_files: a space-separated list
178 # all_docs_url: so we link from doc/foo.html -> doc/
179
180 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
181
182 doctools/split_doc.py \
183 -v build_timestamp="$TIMESTAMP" \
184 -v oil_version="$OIL_VERSION" \
185 -v css_files="$css_files" \
186 -v all_docs_url='.' \
187 -v repo_url="$src" \
188 $src $tmp_prefix
189
190 #ls -l _tmp/doc
191 #head _tmp/doc/*
192 #return
193
194 # for ysh-tour code blocks
195 local code_out=_tmp/code-blocks/$rel_path.txt
196 mkdir -v -p $(dirname $code_out)
197
198 cmark \
199 --code-block-output $code_out \
200 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
201
202 log "$tmp_prefix -> (doctools/cmark) -> $out"
203}
204
205render-from-kate() {
206 ### Make it easier to configure Kate editor
207
208 # It want to pass an absolute path
209 # TODO: I can't figure out how to run this from Kate?
210
211 local full_path=$1
212
213 case $full_path in
214 $REPO_ROOT/*)
215 rel_path=${full_path#"$REPO_ROOT/"}
216 echo "relative path = $rel_path"
217 ;;
218 *)
219 die "$full_path should start with repo root $REPO_ROOT"
220 ;;
221 esac
222
223 split-and-render $rel_path
224}
225
226# Special case for README
227# Do NOT split because we don't want front matter in the markdown source.
228render-only() {
229 local src=${1:-README.md}
230
231 local name
232 case $src in
233 *.md)
234 name=$(basename $src .md)
235 ;;
236 *.txt)
237 name=$(basename $src .txt)
238 ;;
239 *)
240 name=$(basename $src)
241 ;;
242 esac
243
244 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
245 local css_files=${3:-'../web/manual.css ../web/toc.css'}
246 local title=${4:-'Oils Source Code'}
247
248 local prefix=_tmp/doc/$name
249
250 local meta=${prefix}_meta.json
251 cat >$meta <<EOF
252{ "title": "$title",
253 "repo_url": "$src",
254 "css_files": "$css_files",
255 "all_docs_url": ".",
256
257 "build_timestamp": "$TIMESTAMP",
258 "oil_version": "$OIL_VERSION"
259}
260EOF
261
262 cmark $meta $src > $out
263 log "Wrote $out"
264}
265
266special() {
267 # TODO: do all READMEs
268 split-and-render mycpp/README.md \
269 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
270 ../../../web
271
272 local web_dir='../../web'
273 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
274 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
275
276 local web_dir='../web'
277 render-only INSTALL.txt '' \
278 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
279
280 render-only INSTALL-old.txt '' \
281 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
282
283 # These pages aren't in doc/
284 split-and-render doc/release-index.md _tmp/release-index.html
285 split-and-render doc/release-quality.md _tmp/release-quality.html
286}
287
288all-markdown() {
289 make-dirs
290
291 # TODO: We can set repo_url here! Then we don't need it for most docs.
292 # split_doc.py can return {} if the doc doesn't start with ---
293
294 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
295 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
296 for d in "${MARKDOWN_DOCS[@]}"; do
297 split-and-render doc/$d.md
298 done
299
300 special
301}
302
303redir-body() {
304 local to_url=$1 # WARNING: no escaping
305 cat <<EOF
306<head>
307 <meta http-equiv="Refresh" content="0; URL=$to_url" />
308</head>
309EOF
310}
311
312redirect-pairs() {
313 # we want want /release/latest/ URLs to still work
314 cat <<EOF
315oil-language-tour ysh-tour
316oil-language-faq ysh-faq
317oil-help ysh-help
318oil-help-topics ysh-help-topics
319ysh-help ref/toc-ysh
320ysh-help-topics ref/toc-ysh
321EOF
322}
323
324all-redirects() {
325 redirect-pairs | while read -r from_page to_page; do
326 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
327 done
328}
329
330# TODO: This could use some CSS.
331man-page() {
332 local root_dir=${1:-_release/VERSION}
333 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
334 ls -l $root_dir
335}
336
337# I want to ship the INSTALL file literally, so just mutate things
338_sed-ext() {
339 sed --regexp-extended -i "$@"
340}
341
342update-src-versions() {
343 # Update tarball names, etc.
344 _sed-ext \
345 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
346 doc/release-*.md INSTALL.txt INSTALL-old.txt
347
348 # Update /release/0.8.4/ URL, etc.
349 _sed-ext \
350 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
351 doc/osh.1
352}
353
354#
355# Test Tools
356#
357
358split-doc-demo() {
359 cat > _tmp/testdoc.md <<EOF
360---
361title: foo
362---
363
364Title
365=====
366
367hello
368
369EOF
370
371 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
372
373 head _tmp/testdoc*
374}
375
376#
377# Help is both markdown and text
378#
379
380readonly TMP_DIR=_tmp/doc
381readonly CODE_BLOCK_DIR=_tmp/code-blocks
382readonly TEXT_DIR=_devbuild/help
383readonly HTML_DIR=_release/VERSION
384readonly CODE_DIR=_devbuild/gen
385
386cards-from-indices() {
387 ### Make help cards
388
389 for lang in osh ysh data; do
390 help-gen cards-from-index $lang $TEXT_DIR \
391 < $HTML_DIR/doc/ref/toc-$lang.html
392 done
393}
394
395cards-from-chapters() {
396 ### Turn h3 topics into cards
397
398 local py_out=$CODE_DIR/help_meta.py
399
400 mkdir -p _gen/frontend
401 local cc_prefix=_gen/frontend/help_meta
402
403 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
404 $HTML_DIR/doc/ref/chap-*.html
405}
406
407ref-check() {
408 help-gen ref-check \
409 doc/ref/toc-*.md \
410 _release/VERSION/doc/ref/chap-*.html
411}
412
413
414write-metrics() {
415 ### Check indexes and chapters against each other
416
417 local out=_release/VERSION/doc/metrics.txt
418
419 # send stderr to the log file too
420 ref-check > $out 2>&1
421
422 echo "Wrote $out"
423}
424
425tour() {
426 ### Build the Tour of YSH, and execute code as validation
427 local name=${1:-ysh-tour}
428
429 split-and-render doc/$name.md
430
431 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
432
433 mkdir -p $work_dir/lib
434
435 # Files used by module example
436 touch $work_dir/{build,test}.sh
437
438 cat >$work_dir/lib/util.ysh <<EOF
439log() { echo "$@" 1>&2; }
440EOF
441
442 pushd $work_dir
443 $REPO_ROOT/bin/ysh $name.txt
444 popd
445
446 # My own dev tools
447 # if test -d ~/vm-shared; then
448 if false; then
449 local path=_release/VERSION/doc/$name.html
450 cp -v $path ~/vm-shared/$path
451 fi
452}
453
454one() {
455 ### Iterate on one doc quickly
456
457 local name=${1:-options}
458
459 split-and-render doc/$name.md
460
461 # Make sure the doc has valid YSH code?
462 # TODO: Maybe need an attribute for OSH or YSH
463 pushd _tmp/code-blocks/doc
464 $REPO_ROOT/bin/ysh $name.txt
465 popd
466
467 if test -d ~/vm-shared; then
468 local out="${name%.md}.html"
469 local path=_release/VERSION/$out
470 cp -v $path ~/vm-shared/$path
471 fi
472}
473
474make-dirs() {
475 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
476}
477
478one-ref() {
479 local md=${1:-doc/ref/index.md}
480 split-and-render $md '' '../../web'
481}
482
483all-ref() {
484 ### Build doc/ref in text and HTML. Depends on libcmark.so
485
486 log "Removing $TEXT_DIR/*"
487 rm -f $TEXT_DIR/*
488 make-dirs
489
490 # Make the indexes and chapters
491 for d in doc/ref/*.md; do
492 split-and-render $d '' '../../web'
493 done
494
495 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
496 # extract topics from all chapters first, and then make help_meta.json, like
497 # we have _devbuild/gen/help_meta.py.
498
499 # Text cards
500 cards-from-indices
501 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
502 cards-from-chapters
503
504 if command -v pysum; then
505 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
506 # compress it.
507 echo 'Size of embedded help:'
508 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
509 fi
510
511 # Better sorting
512 #LANG=C ls -l $TEXT_DIR
513}
514
515_copy-path() {
516 local src=$1 dest=$2
517 mkdir -p $(dirname $dest)
518 cp -v $src $dest
519}
520
521copy-web() {
522 find web \
523 \( -name _tmp -a -prune \) -o \
524 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
525 xargs -n 2 -- $0 _copy-path
526}
527
528pretty-size() {
529 local path=$1
530 stat --format '%s' "$path" | python -c '
531import sys
532num_bytes = int(sys.stdin.read())
533print "{:,}".format(num_bytes)
534'
535}
536
537# NOTE: It might be better to link to files like this in the /release/ tree.
538# Although I am not signing them.
539
540# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
541
542tarball-links-row-html() {
543 local version=$1
544
545 cat <<EOF
546<tr class="file-table-heading">
547 <td></td>
548 <td>File / SHA256 checksum</td>
549 <td class="size">Size</td>
550 <td></td>
551</tr>
552EOF
553
554 # we switched to .gz for oils-for-unix
555 # note: legacy names for old releases
556 for name in \
557 oils-for-unix-$version.tar.{gz,xz} \
558 oil-$version.tar.{gz,xz} \
559 oil-native-$version.tar.xz; do
560
561 local url="/download/$name" # The server URL
562 local path="../oilshell.org__deploy/download/$name"
563
564 # Don't show tarballs that don't exist
565 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
566 continue
567 fi
568 if [[ $name == oil-native-* && ! -f $path ]]; then
569 continue
570 fi
571
572 local checksum
573 checksum=$(sha256sum $path | awk '{print $1}')
574 local size
575 size=$(pretty-size $path)
576
577 # TODO: Port this to oil with "commas" extension.
578
579 # Three columns: date, version, and links
580 cat <<EOF
581 <tr>
582 <td></td>
583 <td class="filename"><a href="$url">$name</a></td>
584 <td class="size">$size</td>
585 </tr>
586 <tr>
587 <td></td>
588 <td colspan=2 class="checksum">$checksum</td>
589 </tr>
590EOF
591 done
592}
593
594this-release-links() {
595 echo '<div class="file-table">'
596 echo '<table>'
597 tarball-links-row-html "$OIL_VERSION"
598 echo '</table>'
599 echo '</div>'
600}
601
602# Turn HTML comment into a download link
603add-date-and-links() {
604 local snippet
605 snippet=$(this-release-links)
606
607 awk -v date=$1 -v snippet="$snippet" '
608 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
609 print(snippet)
610 next
611 }
612
613 /<!-- REPLACE_WITH_DATE -->/ {
614 print(date)
615 next
616 }
617
618 # Everything else
619 { print }
620 '
621}
622
623patch-release-pages() {
624 local release_date
625 release_date=$(cat _build/release-date.txt)
626
627 local root=_release/VERSION
628
629 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
630 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
631}
632
633copy-release-pages() {
634 ### For testing without releasing
635
636 cat < _tmp/release-index.html > $root/index.html
637 cat < _tmp/release-quality.html > $root/quality.html
638}
639
640run-for-release() {
641 ### Build a tree. Requires _build/release-date.txt to exist
642
643 local root=_release/VERSION
644 mkdir -p $root/{doc,test,pub}
645
646 tour
647
648 # Metadata
649 cp -v _build/release-date.txt oil-version.txt $root
650
651 # Docs
652 # Writes _release/VERSION and _tmp/release-index.html
653 all-markdown
654 all-ref
655 all-redirects # backward compat
656
657 patch-release-pages
658
659 write-metrics
660
661 # Problem: You can't preview it without .wwz!
662 # Maybe have local redirects VERSION/test/wild/ to
663 #
664 # Instead of linking, I should compress them all here.
665
666 copy-web
667
668 if command -v tree >/dev/null; then
669 tree $root
670 else
671 find $root
672 fi
673}
674
675soil-run() {
676 build/stamp.sh write-release-date
677
678 run-for-release
679}
680
681"$@"
682