+ g++ -std=c++11 -Wall -I _tmp/micro-syntax -O0 -fsanitize=address -o _tmp/micro-syntax/micro_syntax doctools/micro_syntax.cc + set +o xtrace CXX doctools/micro_syntax.cc Usage: micro-syntax FLAGS* FILE* Recognizes the syntax of each file,, and prints it to stdout. If there are no files, reads stdin. Flags: -h --help This help -l --lang Language: py|cpp|shell|... -t Print tokens as TSV, instead of ANSI color -w Print HTML for the web -m More color, useful for debugging tokens -n --no-comments Omit comments -o --comments-only Only print comments -e --empty-strs Substitute string literals for empty strings --color on off always more ANSI === [1m[35m<stdin>[0;0m === [90m 1[0;0m echo [31m"[0;0m[31mhi $name[0;0m[31m"[0;0m [32m1 lines, 1 significant[0;0m WEB 7:<stdin>,140:<tr><td class=num>1</td><td id=L1 class=line>echo <span class=str>"</span><span class=str>hi $name</span><span class=str>"</span> </td></tr>,36:{"num_lines": 1, "num_sig_lines": 1}, + echo TSV TSV + echo 'echo "hi $name"' + _tmp/micro-syntax/micro_syntax -l shell -t 7:<stdin>,1 Name 0 4 1 WS 4 5 1 Str 5 6 1 Str 6 14 1 Str 14 15 1 WS 15 16 0:,36:{"num_lines": 1, "num_sig_lines": 1},+ echo + echo + echo '"dq"' + _tmp/micro-syntax/micro_syntax -l shell === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"[0;0m[31mdq[0;0m[31m"[0;0m [32m1 lines, 0 significant[0;0m + _tmp/micro-syntax/micro_syntax -l shell configure + wc -l 491 + set +o errexit + _tmp/micro-syntax/micro_syntax -l shell _nonexistent_ZZ Error opening _nonexistent_ZZ: No such file or directory + local status=1 + test 1 -ne 1 + set -o errexit + echo + run-tests + local bin=_tmp/micro-syntax/micro_syntax + build + local variant=asan + case $variant in + cxxflags='-O0 -fsanitize=address' + mkdir -p _tmp/micro-syntax + local cc=doctools/micro_syntax.cc + local h=_tmp/micro-syntax/micro_syntax.h + local bin=_tmp/micro-syntax/micro_syntax + my-re2c doctools/micro_syntax.re2c.h _tmp/micro-syntax/micro_syntax.h + local in=doctools/micro_syntax.re2c.h + local out=_tmp/micro-syntax/micro_syntax.h + re2c --tags -W -Wno-match-empty-string -Werror -o _tmp/micro-syntax/micro_syntax.h doctools/micro_syntax.re2c.h + set -o xtrace + g++ -std=c++11 -Wall -I _tmp/micro-syntax -O0 -fsanitize=address -o _tmp/micro-syntax/micro_syntax doctools/micro_syntax.cc + set +o xtrace CXX doctools/micro_syntax.cc ==== echo $'multi \n sq \' line' === [1m[35m<stdin>[0;0m === [90m 1[0;0m echo [31m$'[0;0m[31mmulti \n [0;0m[90m 2[0;0m [31m sq \' line[0;0m[31m'[0;0m [32m2 lines, 1 significant[0;0m ==== echo hi \' there === [1m[35m<stdin>[0;0m === [90m 1[0;0m echo hi [31m\'[0;0m there [32m1 lines, 1 significant[0;0m ==== echo one#two === [1m[35m<stdin>[0;0m === [90m 1[0;0m echo one#two [32m1 lines, 1 significant[0;0m ==== echo $(( 16#ff )) === [1m[35m<stdin>[0;0m === [90m 1[0;0m echo $(( 16#ff )) [32m1 lines, 1 significant[0;0m ==== # comment === [1m[35m<stdin>[0;0m === [90m 1[0;0m [34m# comment [0;0m[32m1 lines, 0 significant[0;0m ==== ### comment === [1m[35m<stdin>[0;0m === [90m 1[0;0m [34m### comment [0;0m[32m1 lines, 0 significant[0;0m ==== echo one # comment === [1m[35m<stdin>[0;0m === [90m 1[0;0m echo one [34m# comment [0;0m[32m1 lines, 1 significant[0;0m ==== cat <<EOF hello $world EOF === [1m[35m<stdin>[0;0m === [90m 1[0;0m cat [91m<<EOF[0;0m [90m 2[0;0m [31mhello $world [0;0m[90m 3[0;0m [91mEOF [0;0m[32m3 lines, 1 significant[0;0m ==== cat <<- "EOF" $3.99 EOF === [1m[35m<stdin>[0;0m === [90m 1[0;0m cat [91m<<- "EOF"[0;0m [90m 2[0;0m [31m$3.99 [0;0m[90m 3[0;0m [91mEOF [0;0m[32m3 lines, 1 significant[0;0m ==== cat <<- \_ACAWK $3.99 more _ACAWK echo yo === [1m[35m<stdin>[0;0m === [90m 1[0;0m cat [91m<<- \_ACAWK[0;0m [90m 2[0;0m [31m$3.99 [0;0m[90m 3[0;0m [31mmore [0;0m[90m 4[0;0m [91m_ACAWK [0;0m[90m 5[0;0m echo yo [32m5 lines, 2 significant[0;0m ==== echo multiple << EOF1 << EOF2 > out one EOF1 ... two EOF2 echo done === [1m[35m<stdin>[0;0m === [90m 1[0;0m echo multiple [91m<< EOF1[0;0m [91m<< EOF2[0;0m > out [90m 2[0;0m [31mone [0;0m[90m 3[0;0m [91mEOF1 [0;0m[90m 4[0;0m [31m... [0;0m[90m 5[0;0m [31mtwo [0;0m[90m 6[0;0m [91mEOF2 [0;0m[90m 7[0;0m echo done [32m7 lines, 2 significant[0;0m ==== #if 0 === [1m[35m<stdin>[0;0m === [90m 1[0;0m [35m#if[0;0m 0 [32m1 lines, 1 significant[0;0m ==== not prepreproc #ifdef 0 === [1m[35m<stdin>[0;0m === [90m 1[0;0m not prepreproc #ifdef 0 [32m1 lines, 1 significant[0;0m ==== // comment can't === [1m[35m<stdin>[0;0m === [90m 1[0;0m [34m// comment can't [0;0m[32m1 lines, 0 significant[0;0m ==== f(); // comment isn't === [1m[35m<stdin>[0;0m === [90m 1[0;0m f(); [34m// comment isn't [0;0m[32m1 lines, 1 significant[0;0m ==== '\'' === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m'\''[0;0m [32m1 lines, 0 significant[0;0m ==== void f(); /* multi-line comment */ void g(int x); === [1m[35m<stdin>[0;0m === [90m 1[0;0m void f(); [34m/*[0;0m[34m multi-line [0;0m[90m 2[0;0m [34m comment [0;0m[90m 3[0;0m [34m [0;0m[34m*/[0;0m [90m 4[0;0m void g(int x); [32m4 lines, 2 significant[0;0m ==== #include "foo.h" === [1m[35m<stdin>[0;0m === [90m 1[0;0m [35m#include[0;0m "foo.h" [32m1 lines, 1 significant[0;0m ==== #include <foo.h> // comment === [1m[35m<stdin>[0;0m === [90m 1[0;0m [35m#include[0;0m <foo.h> [34m// comment [0;0m[32m1 lines, 1 significant[0;0m ==== #define X 3 // comment int g(); === [1m[35m<stdin>[0;0m === [90m 1[0;0m [35m#define[0;0m X 3 [34m// comment [0;0m[90m 2[0;0m int g(); [32m2 lines, 2 significant[0;0m ==== // hello #include <stdio.h> #define SUM(x, y) \ (x) + \ (y) // comment void f(); === [1m[35m<stdin>[0;0m === [90m 1[0;0m [34m// hello [0;0m[90m 2[0;0m [35m #include[0;0m <stdio.h> [90m 3[0;0m [35m #define[0;0m SUM(x, y) [35m\ [0;0m[90m 4[0;0m (x) + [35m\ [0;0m[90m 5[0;0m (y) [34m// comment [0;0m[90m 6[0;0m void f(); [32m6 lines, 5 significant[0;0m ==== #undef x === [1m[35m<stdin>[0;0m === [90m 1[0;0m [35m#undef[0;0m x [32m1 lines, 1 significant[0;0m ==== #define F(x) x##name === [1m[35m<stdin>[0;0m === [90m 1[0;0m [35m#define[0;0m F(x) x##name [32m1 lines, 1 significant[0;0m ==== char* s = f(R"(one two three)"); === [1m[35m<stdin>[0;0m === [90m 1[0;0m char* s = f([91mR"([0;0m[31mone [0;0m[90m 2[0;0m [31m two [0;0m[90m 3[0;0m [31m three[0;0m[91m)"[0;0m); [90m 4[0;0m [32m4 lines, 2 significant[0;0m ==== char* s = f(R"zzXX(hi world )zzX" (not the end) )zzXX"); === [1m[35m<stdin>[0;0m === [90m 1[0;0m char* s = f([91mR"zzXX([0;0m[31mhi [0;0m[90m 2[0;0m [31m world [0;0m[90m 3[0;0m [31m [0;0m[31m)zzX"[0;0m[31m (not the end[0;0m[31m)[0;0m[31m [0;0m[90m 4[0;0m [31m [0;0m[91m)zzXX"[0;0m); [90m 5[0;0m [32m5 lines, 2 significant[0;0m ==== char* unclosed = f(R"zzXX(hi world )oops"); === [1m[35m<stdin>[0;0m === [90m 1[0;0m char* unclosed = f([91mR"zzXX([0;0m[31mhi [0;0m[90m 2[0;0m [31m world [0;0m[90m 3[0;0m [31m [0;0m[31m)oops"[0;0m[31m)[0;0m[31m; [0;0m[90m 4[0;0m [31m [0;0m[32m4 lines, 1 significant[0;0m ==== abc === [1m[35m<stdin>[0;0m === [90m 1[0;0m abc [32m1 lines, 1 significant[0;0m ==== "" === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m""[0;0m [32m1 lines, 0 significant[0;0m ==== "dq \" backslash \\" === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"dq \" backslash \\"[0;0m [32m1 lines, 0 significant[0;0m ==== "missing === [1m[35m<stdin>[0;0m === [90m 1[0;0m [7m[31m"[0;0mmissing [32m1 lines, 1 significant[0;0m ==== 'sq \' backslash \\' === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m'sq \' backslash \\'[0;0m [32m1 lines, 0 significant[0;0m ==== "line\n" === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"line\n"[0;0m [32m1 lines, 0 significant[0;0m ==== "quote \" backslash \\ " === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"quote \" backslash \\ "[0;0m [32m1 lines, 0 significant[0;0m ==== "\n" === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"\n"[0;0m [32m1 lines, 0 significant[0;0m ==== hi # comment === [1m[35m<stdin>[0;0m === [90m 1[0;0m hi [34m# comment [0;0m[32m1 lines, 1 significant[0;0m ==== "hi" # comment === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"hi"[0;0m [34m# comment [0;0m[32m1 lines, 0 significant[0;0m ==== (r"raw dq") === [1m[35m<stdin>[0;0m === [90m 1[0;0m ([31mr"raw dq"[0;0m) [32m1 lines, 1 significant[0;0m ==== (r'raw \' sq') === [1m[35m<stdin>[0;0m === [90m 1[0;0m ([31mr'raw \' sq'[0;0m) [32m1 lines, 1 significant[0;0m ==== "L1" # first L2 # second === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"L1"[0;0m [34m# first [0;0m[90m 2[0;0m L2 [34m# second [0;0m[32m2 lines, 1 significant[0;0m ==== def f(): """docstring with "quote" """ pass === [1m[35m<stdin>[0;0m === [90m 1[0;0m def f(): [90m 2[0;0m [31m"""[0;0m[31mdocstring [0;0m[90m 3[0;0m [31m with [0;0m[31m"[0;0m[31mquote[0;0m[31m"[0;0m[31m [0;0m[90m 4[0;0m [31m [0;0m[31m"""[0;0m [90m 5[0;0m pass [32m5 lines, 2 significant[0;0m ==== def f(): '''docstring with 'quote' ''' pass === [1m[35m<stdin>[0;0m === [90m 1[0;0m def f(): [90m 2[0;0m [31m'''[0;0m[31mdocstring [0;0m[90m 3[0;0m [31m with [0;0m[31m'[0;0m[31mquote[0;0m[31m'[0;0m[31m [0;0m[90m 4[0;0m [31m [0;0m[31m'''[0;0m [90m 5[0;0m pass [32m5 lines, 2 significant[0;0m ==== print(r'''hello''') === [1m[35m<stdin>[0;0m === [90m 1[0;0m print([31mr'''[0;0m[31mhello[0;0m[31m'''[0;0m) [32m1 lines, 1 significant[0;0m ==== print(r"""hi there""") === [1m[35m<stdin>[0;0m === [90m 1[0;0m print([31mr"""[0;0m[31mhi there[0;0m[31m"""[0;0m) [32m1 lines, 1 significant[0;0m ==== "hi" # comment === [1m[35m<stdin>[0;0m === [90m 1[0;0m [31m"hi"[0;0m [34m# comment [0;0m[32m1 lines, 0 significant[0;0m ==== f() # hello === [1m[35m<stdin>[0;0m === [90m 1[0;0m f() [34m# hello [0;0m[32m1 lines, 1 significant[0;0m ==== x = f("1 2 \"quote\" 3") === [1m[35m<stdin>[0;0m === [90m 1[0;0m x = f([31m"[0;0m[31m1 [0;0m[90m 2[0;0m [31m 2 \"quote\" [0;0m[90m 3[0;0m [31m 3[0;0m[31m"[0;0m) [32m3 lines, 2 significant[0;0m ==== x = f('1 2 3') === [1m[35m<stdin>[0;0m === [90m 1[0;0m x = f([31m'[0;0m[31m1 [0;0m[90m 2[0;0m [31m 2 [0;0m[90m 3[0;0m [31m 3[0;0m[31m'[0;0m) [32m3 lines, 2 significant[0;0m ==== No language === [1m[35m<stdin>[0;0m === [90m 1[0;0m #!/usr/bin/env bash [90m 2[0;0m # [90m 3[0;0m # Lexing / Parsing experiment [90m 4[0;0m # [90m 5[0;0m # Usage: [90m 6[0;0m # doctools/micro-syntax.sh <function name> [90m 7[0;0m [90m 8[0;0m # TODO: [90m 9[0;0m # - Rename to micro-syntax, from micro-grammars and uchex? [90m 10[0;0m # - micro-segmenting and lexing - comments, strings, and maybe { } [32m10 lines, 9 significant[0;0m /dev/null === [1m[35m<stdin>[0;0m === [32m0 lines, 0 significant[0;0m