OILS / build / doc.sh View on Github | oilshell.org

665 lines, 358 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14
15readonly OIL_VERSION=$(head -n 1 oil-version.txt)
16export OIL_VERSION # for quick_ref.py
17
18THIS_DIR=$(readlink -f $(dirname $0))
19readonly THIS_DIR
20REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
21readonly REPO_ROOT
22
23
24readonly HTML_BASE_DIR=_release/VERSION
25
26
27log() {
28 echo "$@" 1>&2
29}
30
31#
32# Deps (similar to doctools/cmark.sh and build/codegen.sh)
33#
34
35readonly MANDOC_DIR='_deps/mdocml-1.14.1'
36
37download-mandoc() {
38 mkdir -p _deps
39 wget --no-clobber --directory _deps \
40 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
41}
42
43build-mandoc() {
44 cd $MANDOC_DIR
45 ./configure
46 make
47}
48
49mandoc() {
50 $MANDOC_DIR/mandoc "$@"
51}
52
53_build-timestamp() {
54 echo '<hr/>'
55 echo "<i>Generated on $(date)</i>"
56}
57
58# Places version is used
59#
60# - in --version
61# - in URL for every page? inside the binary
62# - in titles for index, install, osh-quick-ref TOC, etc.
63# - in deployment script
64
65# Run with environment variable
66help-gen() {
67 PYTHONPATH=. doctools/help_gen.py "$@"
68}
69
70cmark() {
71 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
72 PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
73}
74
75readonly MARKDOWN_DOCS=(
76 # Help index has its own rendering
77
78 # polished
79 getting-started
80 known-differences
81 error-handling
82 error-catalog
83 json
84 hay
85 simple-word-eval
86 quirks
87 warts
88
89 eggex
90 ysh-regex-api
91 upgrade-breakage
92 ysh-tour
93
94 style-guide
95 novelties
96
97 proc-func
98 block-literals
99
100 # Data language
101 qsn
102 qtt
103 j8-notation
104 pretty-printing
105
106 doc-toolchain
107 doc-plugins
108 idioms
109 shell-idioms
110 ysh-faq
111
112 language-influences
113 ysh-vs-python
114 ysh-vs-shell
115
116 syntactic-concepts
117 syntax-feelings
118 command-vs-expression-mode
119
120 # needs polish
121 # Note: docs about the YSH are prefixed 'ysh-'.
122 # data-model and command-vs-expression-mode span both OSH and YSH
123
124 index
125 faq-doc
126
127 options
128
129 old/index
130 old/project-tour
131 old/legacy-array
132 old/ysh-keywords
133 old/modules
134 old/expression-language
135 old/word-language
136 old/errors
137 old/ysh-builtins
138
139 io-builtins
140 unicode
141 framing
142 xtrace
143 headless
144 completion
145 strings
146 variables
147
148 # Internal stuff
149 interpreter-state
150 process-model
151 architecture-notes
152 parser-architecture
153)
154
155# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
156# locale), which is loaded by Python into say u'\u5e74'. But the default
157# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
158# fail.
159#
160# I believe --rfc-e-mail should never output a Unicode character.
161#
162# A better fix would be to implement json_utf8.load(f), which doesn't decode
163# into unicode instances. This would remove useless conversions.
164
165readonly TIMESTAMP=$(date --rfc-email)
166
167split-and-render() {
168 local src=${1:-doc/known-differences.md}
169
170 local rel_path=${src%'.md'} # doc/known-differences
171 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
172
173 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
174 local web_url=${3:-'../web'}
175
176 mkdir -v -p $(dirname $out) $tmp_prefix
177
178 # Also add could add css_files. The one in the file takes precedence always?
179
180 # css_files: a space-separated list
181 # all_docs_url: so we link from doc/foo.html -> doc/
182
183 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
184
185 doctools/split_doc.py \
186 -v build_timestamp="$TIMESTAMP" \
187 -v oil_version="$OIL_VERSION" \
188 -v css_files="$css_files" \
189 -v all_docs_url='.' \
190 -v repo_url="$src" \
191 $src $tmp_prefix
192
193 #ls -l _tmp/doc
194 #head _tmp/doc/*
195 #return
196
197 # for ysh-tour code blocks
198 local code_out=_tmp/code-blocks/$rel_path.txt
199 mkdir -v -p $(dirname $code_out)
200
201 cmark \
202 --code-block-output $code_out \
203 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
204
205 log "$tmp_prefix -> (doctools/cmark) -> $out"
206}
207
208render-from-kate() {
209 ### Make it easier to configure Kate editor
210
211 # It want to pass an absolute path
212 # TODO: I can't figure out how to run this from Kate?
213
214 local full_path=$1
215
216 case $full_path in
217 $REPO_ROOT/*)
218 rel_path=${full_path#"$REPO_ROOT/"}
219 echo "relative path = $rel_path"
220 ;;
221 *)
222 die "$full_path should start with repo root $REPO_ROOT"
223 ;;
224 esac
225
226 split-and-render $rel_path
227}
228
229# Special case for README
230# Do NOT split because we don't want front matter in the markdown source.
231render-only() {
232 local src=${1:-README.md}
233
234 local name
235 case $src in
236 *.md)
237 name=$(basename $src .md)
238 ;;
239 *.txt)
240 name=$(basename $src .txt)
241 ;;
242 *)
243 name=$(basename $src)
244 ;;
245 esac
246
247 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
248 local css_files=${3:-'../web/manual.css ../web/toc.css'}
249 local title=${4:-'Oils Source Code'}
250
251 local prefix=_tmp/doc/$name
252
253 local meta=${prefix}_meta.json
254 cat >$meta <<EOF
255{ "title": "$title",
256 "repo_url": "$src",
257 "css_files": "$css_files",
258 "all_docs_url": ".",
259
260 "build_timestamp": "$TIMESTAMP",
261 "oil_version": "$OIL_VERSION"
262}
263EOF
264
265 cmark $meta $src > $out
266 log "Wrote $out"
267}
268
269special() {
270 # TODO: do all READMEs
271 split-and-render mycpp/README.md \
272 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
273 ../../../web
274
275 local web_dir='../../web'
276 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
277 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
278
279 local web_dir='../web'
280 render-only 'INSTALL.txt' '' \
281 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
282
283 # These pages aren't in doc/
284 split-and-render doc/release-index.md _tmp/release-index.html
285 split-and-render doc/release-quality.md _tmp/release-quality.html
286}
287
288all-markdown() {
289 make-dirs
290
291 # TODO: We can set repo_url here! Then we don't need it for most docs.
292 # split_doc.py can return {} if the doc doesn't start with ---
293
294 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
295 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
296 for d in "${MARKDOWN_DOCS[@]}"; do
297 split-and-render doc/$d.md
298 done
299
300 special
301}
302
303redir-body() {
304 local to_url=$1 # WARNING: no escaping
305 cat <<EOF
306<head>
307 <meta http-equiv="Refresh" content="0; URL=$to_url" />
308</head>
309EOF
310}
311
312redirect-pairs() {
313 # we want want /release/latest/ URLs to still work
314 cat <<EOF
315oil-language-tour ysh-tour
316oil-language-faq ysh-faq
317oil-help ysh-help
318oil-help-topics ysh-help-topics
319EOF
320}
321
322all-redirects() {
323 redirect-pairs | while read -r from_page to_page; do
324 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
325 done
326}
327
328# TODO: This could use some CSS.
329man-page() {
330 local root_dir=${1:-_release/VERSION}
331 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
332 ls -l $root_dir
333}
334
335# I want to ship the INSTALL file literally, so just mutate things
336_sed-ext() {
337 sed --regexp-extended -i "$@"
338}
339
340update-src-versions() {
341 _sed-ext \
342 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
343 doc/release-*.md
344
345 # we need to update tarball paths, /release/0.8.4/ URL, etc.
346 _sed-ext \
347 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" INSTALL.txt
348
349 _sed-ext \
350 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
351 doc/osh.1
352}
353
354#
355# Test Tools
356#
357
358split-doc-demo() {
359 cat > _tmp/testdoc.md <<EOF
360---
361title: foo
362---
363
364Title
365=====
366
367hello
368
369EOF
370
371 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
372
373 head _tmp/testdoc*
374}
375
376#
377# Help is both markdown and text
378#
379
380readonly TMP_DIR=_tmp/doc
381readonly CODE_BLOCK_DIR=_tmp/code-blocks
382readonly TEXT_DIR=_devbuild/help
383readonly HTML_DIR=_release/VERSION
384readonly CODE_DIR=_devbuild/gen
385
386cards-from-indices() {
387 ### Make help cards
388
389 for lang in osh ysh data; do
390 help-gen cards-from-index $lang $TEXT_DIR \
391 < $HTML_DIR/doc/ref/toc-$lang.html
392 done
393}
394
395cards-from-chapters() {
396 ### Turn h3 topics into cards
397
398 local py_out=$CODE_DIR/help_meta.py
399
400 mkdir -p _gen/frontend
401 local cc_prefix=_gen/frontend/help_meta
402
403 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
404 $HTML_DIR/doc/ref/chap-*.html
405}
406
407ref-check() {
408 ### Check indexes and chapters against each other
409
410 help-gen ref-check \
411 doc/ref/toc-*.md \
412 _release/VERSION/doc/ref/chap-*.html
413}
414
415tour() {
416 ### Build the Tour of YSH, and execute code as validation
417 local name=${1:-ysh-tour}
418
419 split-and-render doc/$name.md
420
421 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
422
423 mkdir -p $work_dir/lib
424
425 # Files used by module example
426 touch $work_dir/{build,test}.sh
427
428 cat >$work_dir/lib/util.ysh <<EOF
429log() { echo "$@" 1>&2; }
430EOF
431
432 pushd $work_dir
433 $REPO_ROOT/bin/ysh $name.txt
434 popd
435
436 # My own dev tools
437 if test -d ~/vm-shared; then
438 local path=_release/VERSION/doc/$name.html
439 cp -v $path ~/vm-shared/$path
440 fi
441}
442
443one() {
444 ### Iterate on one doc quickly
445
446 local name=${1:-options}
447
448 split-and-render doc/$name.md
449
450 # Make sure the doc has valid YSH code?
451 # TODO: Maybe need an attribute for OSH or YSH
452 pushd _tmp/code-blocks/doc
453 $REPO_ROOT/bin/ysh $name.txt
454 popd
455
456 if test -d ~/vm-shared; then
457 local out="${name%.md}.html"
458 local path=_release/VERSION/$out
459 cp -v $path ~/vm-shared/$path
460 fi
461}
462
463make-dirs() {
464 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
465}
466
467one-ref() {
468 local md=${1:-doc/ref/index.md}
469 split-and-render $md '' '../../web'
470}
471
472all-ref() {
473 ### Build doc/ref in text and HTML. Depends on libcmark.so
474
475 log "Removing $TEXT_DIR/*"
476 rm -f $TEXT_DIR/*
477 make-dirs
478
479 # Make the indexes and chapters
480 for d in doc/ref/*.md; do
481 split-and-render $d '' '../../web'
482 done
483
484 # Text cards
485 cards-from-indices
486 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
487 cards-from-chapters
488
489 if command -v pysum; then
490 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
491 # compress it.
492 echo 'Size of embedded help:'
493 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
494 fi
495
496 # Better sorting
497 #LANG=C ls -l $TEXT_DIR
498}
499
500_copy-path() {
501 local src=$1 dest=$2
502 mkdir -p $(dirname $dest)
503 cp -v $src $dest
504}
505
506copy-web() {
507 find web \
508 \( -name _tmp -a -prune \) -o \
509 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
510 xargs -n 2 -- $0 _copy-path
511}
512
513pretty-size() {
514 local path=$1
515 stat --format '%s' "$path" | python -c '
516import sys
517num_bytes = int(sys.stdin.read())
518print "{:,}".format(num_bytes)
519'
520}
521
522# NOTE: It might be better to link to files like this in the /release/ tree.
523# Although I am not signing them.
524
525# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
526
527tarball-links-row-html() {
528 local version=$1
529
530 cat <<EOF
531<tr class="file-table-heading">
532 <td></td>
533 <td>File / SHA256 checksum</td>
534 <td class="size">Size</td>
535 <td></td>
536</tr>
537EOF
538
539 # we switched to .gz for oils-for-unix
540 # note: legacy names for old releases
541 for name in \
542 oils-for-unix-$version.tar.{gz,xz} \
543 oil-$version.tar.{gz,xz} \
544 oil-native-$version.tar.xz; do
545
546 local url="/download/$name" # The server URL
547 local path="../oilshell.org__deploy/download/$name"
548
549 # Don't show tarballs that don't exist
550 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
551 continue
552 fi
553 if [[ $name == oil-native-* && ! -f $path ]]; then
554 continue
555 fi
556
557 local checksum
558 checksum=$(sha256sum $path | awk '{print $1}')
559 local size
560 size=$(pretty-size $path)
561
562 # TODO: Port this to oil with "commas" extension.
563
564 # Three columns: date, version, and links
565 cat <<EOF
566 <tr>
567 <td></td>
568 <td class="filename"><a href="$url">$name</a></td>
569 <td class="size">$size</td>
570 </tr>
571 <tr>
572 <td></td>
573 <td colspan=2 class="checksum">$checksum</td>
574 </tr>
575EOF
576 done
577}
578
579this-release-links() {
580 echo '<div class="file-table">'
581 echo '<table>'
582 tarball-links-row-html "$OIL_VERSION"
583 echo '</table>'
584 echo '</div>'
585}
586
587# Turn HTML comment into a download link
588add-date-and-links() {
589 local snippet
590 snippet=$(this-release-links)
591
592 awk -v date=$1 -v snippet="$snippet" '
593 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
594 print(snippet)
595 next
596 }
597
598 /<!-- REPLACE_WITH_DATE -->/ {
599 print(date)
600 next
601 }
602
603 # Everything else
604 { print }
605 '
606}
607
608patch-release-pages() {
609 local release_date
610 release_date=$(cat _build/release-date.txt)
611
612 local root=_release/VERSION
613
614 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
615 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
616}
617
618copy-release-pages() {
619 ### For testing without releasing
620
621 cat < _tmp/release-index.html > $root/index.html
622 cat < _tmp/release-quality.html > $root/quality.html
623}
624
625run-for-release() {
626 ### Build a tree. Requires _build/release-date.txt to exist
627
628 local root=_release/VERSION
629 mkdir -p $root/{doc,test,pub}
630
631 tour
632
633 # Metadata
634 cp -v _build/release-date.txt oil-version.txt $root
635
636 # Docs
637 # Writes _release/VERSION and _tmp/release-index.html
638 all-markdown
639 all-ref
640 all-redirects # backward compat
641
642 patch-release-pages
643
644 # Problem: You can't preview it without .wwz!
645 # Maybe have local redirects VERSION/test/wild/ to
646 #
647 # Instead of linking, I should compress them all here.
648
649 copy-web
650
651 if command -v tree >/dev/null; then
652 tree $root
653 else
654 find $root
655 fi
656}
657
658soil-run() {
659 build/stamp.sh write-release-date
660
661 run-for-release
662}
663
664"$@"
665