+ g++ -std=c++11 -Wall -I _tmp/micro-syntax -O0 -fsanitize=address -o _tmp/micro-syntax/micro_syntax doctools/micro_syntax.cc + set +o xtrace CXX doctools/micro_syntax.cc Usage: micro-syntax FLAGS* FILE* Recognizes the syntax of each file,, and prints it to stdout. If there are no files, reads stdin. Flags: -h --help This help -l --lang Language: py|cpp|shell|... -t Print tokens as TSV, instead of ANSI color -w Print HTML for the web -m More color, useful for debugging tokens -n --no-comments Omit comments -o --comments-only Only print comments -e --empty-strs Substitute string literals for empty strings --color on off always more ANSI ===  ===  1 echo "hi $name" 1 lines, 1 significant WEB 7:,140:1echo "hi $name" ,36:{"num_lines": 1, "num_sig_lines": 1}, + echo TSV TSV + _tmp/micro-syntax/micro_syntax -l shell -t + echo 'echo "hi $name"' 7:,1 Name 0 4 1 WS 4 5 1 Str 5 6 1 Str 6 14 1 Str 14 15 1 WS 15 16 0:,36:{"num_lines": 1, "num_sig_lines": 1},+ echo + echo + echo '"dq"' + _tmp/micro-syntax/micro_syntax -l shell ===  ===  1 "dq" 1 lines, 0 significant + _tmp/micro-syntax/micro_syntax -l shell configure + wc -l 491 + set +o errexit + _tmp/micro-syntax/micro_syntax -l shell _nonexistent_ZZ Error opening _nonexistent_ZZ: No such file or directory + local status=1 + test 1 -ne 1 + set -o errexit + echo + run-tests + local bin=_tmp/micro-syntax/micro_syntax + build + local variant=asan + case $variant in + cxxflags='-O0 -fsanitize=address' + mkdir -p _tmp/micro-syntax + local cc=doctools/micro_syntax.cc + local h=_tmp/micro-syntax/micro_syntax.h + local bin=_tmp/micro-syntax/micro_syntax + my-re2c doctools/micro_syntax.re2c.h _tmp/micro-syntax/micro_syntax.h + local in=doctools/micro_syntax.re2c.h + local out=_tmp/micro-syntax/micro_syntax.h + re2c --tags -W -Wno-match-empty-string -Werror -o _tmp/micro-syntax/micro_syntax.h doctools/micro_syntax.re2c.h + set -o xtrace + g++ -std=c++11 -Wall -I _tmp/micro-syntax -O0 -fsanitize=address -o _tmp/micro-syntax/micro_syntax doctools/micro_syntax.cc + set +o xtrace CXX doctools/micro_syntax.cc ==== echo $'multi \n sq \' line' ===  ===  1 echo $'multi \n  2  sq \' line' 2 lines, 1 significant ==== echo hi \' there ===  ===  1 echo hi \' there 1 lines, 1 significant ==== echo one#two ===  ===  1 echo one#two 1 lines, 1 significant ==== echo $(( 16#ff )) ===  ===  1 echo $(( 16#ff )) 1 lines, 1 significant ==== # comment ===  ===  1 # comment 1 lines, 0 significant ==== ### comment ===  ===  1 ### comment 1 lines, 0 significant ==== echo one # comment ===  ===  1 echo one # comment 1 lines, 1 significant ==== cat < ===  1 cat < ===  1 cat <<- "EOF"  2 $3.99  3 EOF 3 lines, 1 significant ==== cat <<- \_ACAWK $3.99 more _ACAWK echo yo ===  ===  1 cat <<- \_ACAWK  2 $3.99  3 more  4 _ACAWK  5 echo yo 5 lines, 2 significant ==== echo multiple << EOF1 << EOF2 > out one EOF1 ... two EOF2 echo done ===  ===  1 echo multiple << EOF1 << EOF2 > out  2 one  3 EOF1  4 ...  5 two  6 EOF2  7 echo done 7 lines, 2 significant ==== #if 0 ===  ===  1 #if 0 1 lines, 1 significant ==== not prepreproc #ifdef 0 ===  ===  1 not prepreproc #ifdef 0 1 lines, 1 significant ==== // comment can't ===  ===  1 // comment can't 1 lines, 0 significant ==== f(); // comment isn't ===  ===  1 f(); // comment isn't 1 lines, 1 significant ==== '\'' ===  ===  1 '\'' 1 lines, 0 significant ==== void f(); /* multi-line comment */ void g(int x); ===  ===  1 void f(); /* multi-line  2  comment  3  */  4 void g(int x); 4 lines, 2 significant ==== #include "foo.h" ===  ===  1 #include "foo.h" 1 lines, 1 significant ==== #include // comment ===  ===  1 #include // comment 1 lines, 1 significant ==== #define X 3 // comment int g(); ===  ===  1 #define X 3 // comment  2 int g(); 2 lines, 2 significant ==== // hello #include #define SUM(x, y) \ (x) + \ (y) // comment void f(); ===  ===  1 // hello  2  #include  3  #define SUM(x, y) \  4 (x) + \  5 (y) // comment  6 void f(); 6 lines, 5 significant ==== #undef x ===  ===  1 #undef x 1 lines, 1 significant ==== #define F(x) x##name ===  ===  1 #define F(x) x##name 1 lines, 1 significant ==== char* s = f(R"(one two three)"); ===  ===  1 char* s = f(R"(one  2  two  3  three)");  4 4 lines, 2 significant ==== char* s = f(R"zzXX(hi world )zzX" (not the end) )zzXX"); ===  ===  1 char* s = f(R"zzXX(hi  2  world  3  )zzX" (not the end)  4  )zzXX");  5 5 lines, 2 significant ==== char* unclosed = f(R"zzXX(hi world )oops"); ===  ===  1 char* unclosed = f(R"zzXX(hi  2  world  3  )oops");  4  4 lines, 1 significant ==== abc ===  ===  1 abc 1 lines, 1 significant ==== "" ===  ===  1 "" 1 lines, 0 significant ==== "dq \" backslash \\" ===  ===  1 "dq \" backslash \\" 1 lines, 0 significant ==== "missing ===  ===  1 "missing 1 lines, 1 significant ==== 'sq \' backslash \\' ===  ===  1 'sq \' backslash \\' 1 lines, 0 significant ==== "line\n" ===  ===  1 "line\n" 1 lines, 0 significant ==== "quote \" backslash \\ " ===  ===  1 "quote \" backslash \\ " 1 lines, 0 significant ==== "\n" ===  ===  1 "\n" 1 lines, 0 significant ==== hi # comment ===  ===  1 hi # comment 1 lines, 1 significant ==== "hi" # comment ===  ===  1 "hi" # comment 1 lines, 0 significant ==== (r"raw dq") ===  ===  1 (r"raw dq") 1 lines, 1 significant ==== (r'raw \' sq') ===  ===  1 (r'raw \' sq') 1 lines, 1 significant ==== "L1" # first L2 # second ===  ===  1 "L1" # first  2 L2 # second 2 lines, 1 significant ==== def f(): """docstring with "quote" """ pass ===  ===  1 def f():  2 """docstring  3  with "quote"  4  """  5 pass 5 lines, 2 significant ==== def f(): '''docstring with 'quote' ''' pass ===  ===  1 def f():  2 '''docstring  3  with 'quote'  4  '''  5 pass 5 lines, 2 significant ==== print(r'''hello''') ===  ===  1 print(r'''hello''') 1 lines, 1 significant ==== print(r"""hi there""") ===  ===  1 print(r"""hi there""") 1 lines, 1 significant ==== "hi" # comment ===  ===  1 "hi" # comment 1 lines, 0 significant ==== f() # hello ===  ===  1 f() # hello 1 lines, 1 significant ==== x = f("1 2 \"quote\" 3") ===  ===  1 x = f("1  2  2 \"quote\"  3  3") 3 lines, 2 significant ==== x = f('1 2 3') ===  ===  1 x = f('1  2  2  3  3') 3 lines, 2 significant ==== No language ===  ===  1 #!/usr/bin/env bash  2 #  3 # Lexing / Parsing experiment  4 #  5 # Usage:  6 # doctools/micro-syntax.sh  7  8 # TODO:  9 # - Rename to micro-syntax, from micro-grammars and uchex?  10 # - micro-segmenting and lexing - comments, strings, and maybe { } 10 lines, 9 significant /dev/null ===  === 0 lines, 0 significant