OILS / build / doc.sh View on Github | oilshell.org

695 lines, 369 significant
1#!/usr/bin/env bash
2#
3# Usage:
4# build/doc.sh <function name>
5
6set -o nounset
7set -o pipefail
8set -o errexit
9
10# https://oilshell.org/release/$VERSION/
11# doc/
12# index.html
13# INSTALL.html
14# INSTALL-old.html
15
16readonly OIL_VERSION=$(head -n 1 oil-version.txt)
17export OIL_VERSION # for quick_ref.py
18
19THIS_DIR=$(readlink -f $(dirname $0))
20readonly THIS_DIR
21REPO_ROOT=$(cd $THIS_DIR/.. && pwd)
22readonly REPO_ROOT
23
24
25readonly HTML_BASE_DIR=_release/VERSION
26
27
28log() {
29 echo "$@" 1>&2
30}
31
32#
33# Deps (similar to doctools/cmark.sh and build/codegen.sh)
34#
35
36readonly MANDOC_DIR='_deps/mdocml-1.14.1'
37
38download-mandoc() {
39 mkdir -p _deps
40 wget --no-clobber --directory _deps \
41 https://mandoc.bsd.lv/snapshots/mdocml-1.14.1.tar.gz
42}
43
44build-mandoc() {
45 cd $MANDOC_DIR
46 ./configure
47 make
48}
49
50mandoc() {
51 $MANDOC_DIR/mandoc "$@"
52}
53
54# Places version is used
55#
56# - in --version
57# - in URL for every page? inside the binary
58# - in titles for index, install, osh-quick-ref TOC, etc.
59# - in deployment script
60
61# Run with environment variable
62help-gen() {
63 PYTHONPATH=. doctools/help_gen.py "$@"
64}
65
66cmark() {
67 # h2 and h3 are shown in TOC. The blog uses "legacy" h3 and h4.
68 PYTHONPATH=. doctools/cmark.py --toc-tag h2 --toc-tag h3 --toc-pretty-href "$@"
69}
70
71readonly MARKDOWN_DOCS=(
72 published
73
74 # polished
75 getting-started
76 portability
77 known-differences
78 ysh-error
79 error-handling
80 error-catalog
81 json
82 hay
83 simple-word-eval
84 quirks
85 warts
86
87 eggex
88 ysh-regex-api
89 upgrade-breakage
90 ysh-tour
91
92 style-guide
93 novelties
94
95 proc-func
96 block-literals
97
98 # Data language
99 qsn
100 qtt
101 j8-notation
102 # Protocol
103 byo
104 pretty-printing
105 stream-table-process
106
107 lib-osh
108
109 doc-toolchain
110 doc-plugins
111 idioms
112 shell-idioms
113 ysh-faq
114
115 language-influences
116 ysh-vs-python
117 ysh-vs-shell
118
119 syntactic-concepts
120 syntax-feelings
121 command-vs-expression-mode
122
123 # needs polish
124 # Note: docs about the YSH are prefixed 'ysh-'.
125 # data-model and command-vs-expression-mode span both OSH and YSH
126
127 index
128 faq-doc
129
130 options
131
132 old/index
133 old/project-tour
134 old/legacy-array
135 old/ysh-keywords
136 old/modules
137 old/expression-language
138 old/word-language
139 old/errors
140 old/ysh-builtins
141
142 io-builtins
143 unicode
144 framing
145 xtrace
146 headless
147 completion
148 strings
149 variables
150
151 # Internal stuff
152 interpreter-state
153 process-model
154 architecture-notes
155 parser-architecture
156)
157
158# Bug fix: Plain $(date) can output unicode characters (e.g. in Japanese
159# locale), which is loaded by Python into say u'\u5e74'. But the default
160# encoding in Python 2 is still 'ascii', which means that '%s' % u_str may
161# fail.
162#
163# I believe --rfc-e-mail should never output a Unicode character.
164#
165# A better fix would be to implement json_utf8.load(f), which doesn't decode
166# into unicode instances. This would remove useless conversions.
167
168readonly TIMESTAMP=$(date --rfc-email)
169
170split-and-render() {
171 local src=${1:-doc/known-differences.md}
172
173 local rel_path=${src%'.md'} # doc/known-differences
174 local tmp_prefix=_tmp/$rel_path # temp dir for splitting
175
176 local out=${2:-$HTML_BASE_DIR/$rel_path.html}
177 local web_url=${3:-'../web'}
178
179 mkdir -v -p $(dirname $out) $tmp_prefix
180
181 # Also add could add css_files. The one in the file takes precedence always?
182
183 # css_files: a space-separated list
184 # all_docs_url: so we link from doc/foo.html -> doc/
185
186 local css_files="$web_url/base.css $web_url/manual.css $web_url/toc.css $web_url/language.css $web_url/code.css"
187
188 doctools/split_doc.py \
189 -v build_timestamp="$TIMESTAMP" \
190 -v oil_version="$OIL_VERSION" \
191 -v css_files="$css_files" \
192 -v all_docs_url='.' \
193 -v repo_url="$src" \
194 $src $tmp_prefix
195
196 #ls -l _tmp/doc
197 #head _tmp/doc/*
198 #return
199
200 # for ysh-tour code blocks
201 local code_out=_tmp/code-blocks/$rel_path.txt
202 mkdir -v -p $(dirname $code_out)
203
204 cmark \
205 --code-block-output $code_out \
206 ${tmp_prefix}_meta.json ${tmp_prefix}_content.md > $out
207
208 log "$tmp_prefix -> (doctools/cmark) -> $out"
209}
210
211render-from-kate() {
212 ### Make it easier to configure Kate editor
213
214 # It want to pass an absolute path
215 # TODO: I can't figure out how to run this from Kate?
216
217 local full_path=$1
218
219 case $full_path in
220 $REPO_ROOT/*)
221 rel_path=${full_path#"$REPO_ROOT/"}
222 echo "relative path = $rel_path"
223 ;;
224 *)
225 die "$full_path should start with repo root $REPO_ROOT"
226 ;;
227 esac
228
229 split-and-render $rel_path
230}
231
232# Special case for README
233# Do NOT split because we don't want front matter in the markdown source.
234render-only() {
235 local src=${1:-README.md}
236
237 local name
238 case $src in
239 *.md)
240 name=$(basename $src .md)
241 ;;
242 *.txt)
243 name=$(basename $src .txt)
244 ;;
245 *)
246 name=$(basename $src)
247 ;;
248 esac
249
250 local out=${2:-$HTML_BASE_DIR/doc/$name.html}
251 local css_files=${3:-'../web/manual.css ../web/toc.css'}
252 local title=${4:-'Oils Source Code'}
253
254 local prefix=_tmp/doc/$name
255
256 local meta=${prefix}_meta.json
257 cat >$meta <<EOF
258{ "title": "$title",
259 "repo_url": "$src",
260 "css_files": "$css_files",
261 "all_docs_url": ".",
262
263 "build_timestamp": "$TIMESTAMP",
264 "oil_version": "$OIL_VERSION"
265}
266EOF
267
268 cmark $meta $src > $out
269 log "Wrote $out"
270}
271
272special() {
273 # TODO: do all READMEs
274 split-and-render mycpp/README.md \
275 $HTML_BASE_DIR/doc/oils-repo/mycpp/README.html \
276 ../../../web
277
278 local web_dir='../../web'
279 render-only 'README.md' $HTML_BASE_DIR/doc/oils-repo/README.html \
280 "$web_dir/base.css $web_dir/manual.css $web_dir/toc.css" 'Oils Source Code'
281
282 local web_dir='../web'
283 render-only INSTALL.txt '' \
284 "$web_dir/base.css $web_dir/install.css" 'Installing Oils'
285
286 render-only INSTALL-old.txt '' \
287 "$web_dir/base.css $web_dir/install.css" 'Installing Oils - old CPython build'
288
289 # These pages aren't in doc/
290 split-and-render doc/release-index.md _tmp/release-index.html
291 split-and-render doc/release-quality.md _tmp/release-quality.html
292}
293
294all-markdown() {
295 make-dirs
296
297 # TODO: We can set repo_url here! Then we don't need it for most docs.
298 # split_doc.py can return {} if the doc doesn't start with ---
299
300 #for d in doc/index.md doc/known-differences.md doc/*-manual.md \
301 # doc/eggex.md doc/oil-options.md doc/oil-func-proc-block.md; do
302 for d in "${MARKDOWN_DOCS[@]}"; do
303 split-and-render doc/$d.md
304 done
305
306 special
307}
308
309redir-body() {
310 local to_url=$1 # WARNING: no escaping
311 cat <<EOF
312<head>
313 <meta http-equiv="Refresh" content="0; URL=$to_url" />
314</head>
315EOF
316}
317
318redirect-pairs() {
319 # we want want /release/latest/ URLs to still work
320 cat <<EOF
321oil-language-tour ysh-tour
322oil-language-faq ysh-faq
323oil-help ysh-help
324oil-help-topics ysh-help-topics
325ysh-help ref/toc-ysh
326ysh-help-topics ref/toc-ysh
327EOF
328}
329
330all-redirects() {
331 redirect-pairs | while read -r from_page to_page; do
332 redir-body "$to_page.html" | tee "_release/VERSION/doc/$from_page.html"
333 done
334}
335
336# TODO: This could use some CSS.
337man-page() {
338 local root_dir=${1:-_release/VERSION}
339 mandoc -T html doc/osh.1 > $root_dir/osh.1.html
340 ls -l $root_dir
341}
342
343# I want to ship the INSTALL file literally, so just mutate things
344_sed-ext() {
345 sed --regexp-extended -i "$@"
346}
347
348update-src-versions() {
349 # Update tarball names, etc.
350 _sed-ext \
351 "s/[0-9]+\.[0-9]+\.[a-z0-9]+/$OIL_VERSION/g" \
352 doc/release-*.md INSTALL.txt INSTALL-old.txt
353
354 # Update /release/0.8.4/ URL, etc.
355 _sed-ext \
356 "s;/release/[0-9]+\.[0-9]+\.[a-z0-9]+/;/release/$OIL_VERSION/;g" \
357 doc/osh.1
358}
359
360#
361# Test Tools
362#
363
364split-doc-demo() {
365 cat > _tmp/testdoc.md <<EOF
366---
367title: foo
368---
369
370Title
371=====
372
373hello
374
375EOF
376
377 doctools/split_doc.py _tmp/testdoc.md _tmp/testdoc
378
379 head _tmp/testdoc*
380}
381
382#
383# Help is both markdown and text
384#
385
386readonly TMP_DIR=_tmp/doc
387readonly CODE_BLOCK_DIR=_tmp/code-blocks
388readonly TEXT_DIR=_devbuild/help
389readonly HTML_DIR=_release/VERSION
390readonly CODE_DIR=_devbuild/gen
391
392cards-from-indices() {
393 ### Make help cards
394
395 for lang in osh ysh data; do
396 help-gen cards-from-index $lang $TEXT_DIR \
397 < $HTML_DIR/doc/ref/toc-$lang.html
398 done
399}
400
401cards-from-chapters() {
402 ### Turn h3 topics into cards
403
404 local py_out=$CODE_DIR/help_meta.py
405
406 mkdir -p _gen/frontend
407 local cc_prefix=_gen/frontend/help_meta
408
409 help-gen cards-from-chapters $TEXT_DIR $py_out $cc_prefix \
410 $HTML_DIR/doc/ref/chap-*.html
411}
412
413ref-check() {
414 help-gen ref-check \
415 doc/ref/toc-*.md \
416 _release/VERSION/doc/ref/chap-*.html
417}
418
419fmt-check() {
420 PYTHONPATH=. doctools/fmt_check.py _release/VERSION/doc/ref/*.html
421}
422
423
424write-metrics() {
425 ### Check indexes and chapters against each other
426
427 local out=_release/VERSION/doc/metrics.txt
428
429 # send stderr to the log file too
430 ref-check > $out 2>&1
431
432 echo "Wrote $out"
433}
434
435tour() {
436 ### Build the Tour of YSH, and execute code as validation
437 local name=${1:-ysh-tour}
438
439 split-and-render doc/$name.md
440
441 local work_dir=$REPO_ROOT/_tmp/code-blocks/doc
442
443 mkdir -p $work_dir/lib
444
445 # Files used by module example
446 touch $work_dir/{build,test}.sh
447
448 cat >$work_dir/lib/util.ysh <<EOF
449log() { echo "$@" 1>&2; }
450EOF
451
452 pushd $work_dir
453 # Fix: don't supply stdin!
454 $REPO_ROOT/bin/ysh $name.txt < /dev/null
455 popd
456
457 # My own dev tools
458 # if test -d ~/vm-shared; then
459 if false; then
460 local path=_release/VERSION/doc/$name.html
461 cp -v $path ~/vm-shared/$path
462 fi
463}
464
465one() {
466 ### Iterate on one doc quickly
467
468 local name=${1:-options}
469
470 split-and-render doc/$name.md
471
472 # Make sure the doc has valid YSH code?
473 # TODO: Maybe need an attribute for OSH or YSH
474 pushd _tmp/code-blocks/doc
475 $REPO_ROOT/bin/ysh $name.txt
476 popd
477
478 if test -d ~/vm-shared; then
479 local out="${name%.md}.html"
480 local path=_release/VERSION/$out
481 cp -v $path ~/vm-shared/$path
482 fi
483}
484
485make-dirs() {
486 mkdir -p $TMP_DIR $CODE_BLOCK_DIR $TEXT_DIR $HTML_DIR/doc
487}
488
489one-ref() {
490 local md=${1:-doc/ref/index.md}
491 split-and-render $md '' '../../web'
492}
493
494all-ref() {
495 ### Build doc/ref in text and HTML. Depends on libcmark.so
496
497 log "Removing $TEXT_DIR/*"
498 rm -f $TEXT_DIR/*
499 make-dirs
500
501 # Make the indexes and chapters
502 for d in doc/ref/*.md; do
503 split-and-render $d '' '../../web'
504 done
505
506 # Note: if we want a $ref-topic shortcut, we might want to use Ninja to
507 # extract topics from all chapters first, and then make help_meta.json, like
508 # we have _devbuild/gen/help_meta.py.
509
510 # Text cards
511 cards-from-indices
512 # A few text cards, and HELP_TOPICS dict for URLs, for flat namespace
513 cards-from-chapters
514
515 if command -v pysum; then
516 # 19 KB of embedded help, seems OK. Biggest card is 'ysh-option'. Could
517 # compress it.
518 echo 'Size of embedded help:'
519 ls -l $TEXT_DIR | tee /dev/stderr | awk '{print $5}' | pysum
520 fi
521
522 # Better sorting
523 #LANG=C ls -l $TEXT_DIR
524}
525
526_copy-path() {
527 local src=$1 dest=$2
528 mkdir -p $(dirname $dest)
529 cp -v $src $dest
530}
531
532copy-web() {
533 find web \
534 \( -name _tmp -a -prune \) -o \
535 \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
536 xargs -n 2 -- $0 _copy-path
537}
538
539pretty-size() {
540 local path=$1
541 stat --format '%s' "$path" | python -c '
542import sys
543num_bytes = int(sys.stdin.read())
544print "{:,}".format(num_bytes)
545'
546}
547
548# NOTE: It might be better to link to files like this in the /release/ tree.
549# Although I am not signing them.
550
551# https://nodejs.org/dist/v8.11.4/SHASUMS256.txt.asc
552
553tarball-links-row-html() {
554 local version=$1
555
556 cat <<EOF
557<tr class="file-table-heading">
558 <td></td>
559 <td>File / SHA256 checksum</td>
560 <td class="size">Size</td>
561 <td></td>
562</tr>
563EOF
564
565 # we switched to .gz for oils-for-unix
566 # note: legacy names for old releases
567 for name in \
568 oils-for-unix-$version.tar.{gz,xz} \
569 oil-$version.tar.{gz,xz} \
570 oil-native-$version.tar.xz; do
571
572 local url="/download/$name" # The server URL
573 local path="../oilshell.org__deploy/download/$name"
574
575 # Don't show tarballs that don't exist
576 if [[ $name == oils-for-unix-* && ! -f $path ]]; then
577 continue
578 fi
579 if [[ $name == oil-native-* && ! -f $path ]]; then
580 continue
581 fi
582
583 local checksum
584 checksum=$(sha256sum $path | awk '{print $1}')
585 local size
586 size=$(pretty-size $path)
587
588 # TODO: Port this to oil with "commas" extension.
589
590 # Three columns: date, version, and links
591 cat <<EOF
592 <tr>
593 <td></td>
594 <td class="filename"><a href="$url">$name</a></td>
595 <td class="size">$size</td>
596 </tr>
597 <tr>
598 <td></td>
599 <td colspan=2 class="checksum">$checksum</td>
600 </tr>
601EOF
602 done
603}
604
605this-release-links() {
606 echo '<div class="file-table">'
607 echo '<table>'
608 tarball-links-row-html "$OIL_VERSION"
609 echo '</table>'
610 echo '</div>'
611}
612
613# Turn HTML comment into a download link
614add-date-and-links() {
615 local snippet
616 snippet=$(this-release-links)
617
618 awk -v date=$1 -v snippet="$snippet" '
619 /<!-- REPLACE_WITH_DOWNLOAD_LINKS -->/ {
620 print(snippet)
621 next
622 }
623
624 /<!-- REPLACE_WITH_DATE -->/ {
625 print(date)
626 next
627 }
628
629 # Everything else
630 { print }
631 '
632}
633
634patch-release-pages() {
635 local release_date
636 release_date=$(cat _build/release-date.txt)
637
638 local root=_release/VERSION
639
640 add-date-and-links $release_date < _tmp/release-index.html > $root/index.html
641 add-date-and-links $release_date < _tmp/release-quality.html > $root/quality.html
642}
643
644copy-release-pages() {
645 ### For testing without releasing
646
647 cat < _tmp/release-index.html > $root/index.html
648 cat < _tmp/release-quality.html > $root/quality.html
649}
650
651run-for-release() {
652 ### Build a tree. Requires _build/release-date.txt to exist
653
654 local root=_release/VERSION
655 mkdir -p $root/{doc,test,pub}
656
657 tour
658
659 # Metadata
660 cp -v _build/release-date.txt oil-version.txt $root
661
662 # Docs
663 # Writes _release/VERSION and _tmp/release-index.html
664 all-markdown
665 all-ref
666 all-redirects # backward compat
667
668 fmt-check # Needs to run *after* we build the HTML
669
670 patch-release-pages
671
672 write-metrics
673
674 # Problem: You can't preview it without .wwz!
675 # Maybe have local redirects VERSION/test/wild/ to
676 #
677 # Instead of linking, I should compress them all here.
678
679 copy-web
680
681 if command -v tree >/dev/null; then
682 tree $root
683 else
684 find $root
685 fi
686}
687
688soil-run() {
689 build/stamp.sh write-release-date
690
691 run-for-release
692}
693
694"$@"
695