OILS / test / spec-cpp.sh View on Github | oilshell.org

360 lines, 158 significant
1#!/usr/bin/env bash
2#
3# Test the C++ translation of Oils.
4#
5# Usage:
6# test/spec-cpp.sh <function name>
7#
8# Examples:
9# test/spec-cpp.sh run-file smoke -r 0 -v
10# NUM_SPEC_TASKS=2 test/spec-cpp.sh osh-all
11
12: ${LIB_OSH=stdlib/osh}
13source $LIB_OSH/bash-strict.sh
14source $LIB_OSH/task-five.sh
15
16REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
17
18source build/dev-shell.sh # PYTHONPATH
19source test/common.sh # html-head
20source test/spec-common.sh
21source web/table/html.sh
22
23shopt -s failglob # to debug TSV expansion failure below
24
25OSH_PY=$REPO_ROOT/bin/osh
26YSH_PY=$REPO_ROOT/bin/ysh
27
28# Run with ASAN binary by default. Release overrides this
29OSH_CC=${OSH_CC:-$REPO_ROOT/_bin/cxx-asan/osh}
30YSH_CC=${YSH_CC:-$REPO_ROOT/_bin/cxx-asan/ysh}
31
32# Same variable in test/spec-runner.sh
33NUM_SPEC_TASKS=${NUM_SPEC_TASKS:-400}
34
35# So we can pass ASAN. Note that test/spec-common.sh has to pass this to
36# sh_spec.py.
37export OILS_GC_ON_EXIT=1
38
39#
40# For translation
41#
42
43run-file() {
44 local spec_name=$1
45 shift
46
47 local spec_file=spec/$spec_name.test.sh
48
49 local suite
50 suite=$(test/sh_spec.py --print-spec-suite $spec_file)
51
52 local spec_subdir
53 case $suite in
54 osh) spec_subdir='osh-cpp' ;;
55 ysh) spec_subdir='ysh-cpp' ;;
56 *) die "Invalid suite $suite" ;;
57 esac
58
59 local base_dir=_tmp/spec/$spec_subdir
60 mkdir -v -p $base_dir
61
62 # Compare Python and C++ shells by passing --oils-cpp-bin-dir
63 sh-spec $spec_file \
64 --timeout 10 \
65 --oils-bin-dir $PWD/bin \
66 --oils-cpp-bin-dir $REPO_ROOT/_bin/cxx-asan \
67 --tsv-output $base_dir/${spec_name}.tsv \
68 "$@"
69}
70
71osh-all() {
72 # Like test/spec.sh {osh,ysh}-all, but it compares against different binaries
73
74 # For debugging hangs
75 #export MAX_PROCS=1
76
77 ninja _bin/cxx-asan/{osh,ysh}
78
79 test/spec-runner.sh shell-sanity-check $OSH_PY $OSH_CC
80
81 local spec_subdir=osh-cpp
82
83 # $suite $compare_mode
84 test/spec-runner.sh all-parallel \
85 osh compare-cpp $spec_subdir "$@" || true # OK if it fails
86
87 write-compare-html $spec_subdir
88}
89
90ysh-all() {
91 ninja _bin/cxx-asan/{osh,ysh}
92
93 local spec_subdir=ysh-cpp
94
95 # $suite $compare_mode
96 test/spec-runner.sh all-parallel \
97 ysh compare-cpp $spec_subdir "$@" || true # OK if it fails
98
99 write-compare-html $spec_subdir
100}
101
102console-row() {
103 ### Print out a histogram of results
104
105 awk '
106FNR == 1 {
107 #print FILENAME > "/dev/stderr"
108}
109FNR != 1 {
110 case_num = $1
111 sh = $2
112 result = $3
113
114 if (sh == "osh") {
115 osh[result] += 1
116 } else if (sh == "osh_cpp") { # bin/osh_cpp
117 oe_py[result] += 1
118 } else if (sh == "osh_ALT") { # _bin/*/osh
119 oe_cpp[result] += 1
120 }
121}
122
123function print_hist(sh, hist) {
124 printf("%s\t", sh)
125
126 k = "pass"
127 printf("%s %4d\t", k, hist[k])
128 k = "FAIL"
129 printf("%s %4d\t", k, hist[k])
130
131 print ""
132
133 # This prints N-I, ok, bug, etc.
134 #for (k in hist) {
135 # printf("%s %s\t", k, hist[k])
136 #}
137
138}
139
140END {
141 print_hist("osh", osh)
142 print_hist("osh_cpp", oe_py)
143 print_hist("osh_ALT", oe_cpp)
144}
145 ' "$@"
146}
147
148console-summary() {
149 ### Report on our progress translating
150
151 local spec_subdir=$1
152
153 # Can't go at the top level because files won't exist!
154 readonly TSV=(_tmp/spec/$spec_subdir/*.tsv)
155
156 wc -l "${TSV[@]}"
157
158 for file in "${TSV[@]}"; do
159 echo
160 echo "$file"
161 console-row $file
162 done
163
164 echo
165 echo "TOTAL"
166 console-row "${TSV[@]}"
167}
168
169#
170# HTML
171#
172
173summary-csv-row() {
174 ### Print one row or the last total row
175
176 local spec_subdir=$1
177 shift
178
179 if test $# -eq 1; then
180 local spec_name=$1
181 local -a tsv_files=( _tmp/spec/$spec_subdir/$spec_name.tsv )
182 else
183 local spec_name='TOTAL'
184 local -a tsv_files=( "$@" )
185 fi
186
187 awk -v spec_name=$spec_name '
188# skip the first row
189FNR != 1 {
190 case_num = $1
191 sh = $2
192 result = $3
193
194 if (sh == "osh" || sh == "ysh") {
195 osh[result] += 1
196 } else if (sh == "osh-cpp" || sh == "ysh-cpp") { # bin/osh
197 osh_native[result] += 1
198 }
199}
200
201END {
202 num_py = osh["pass"]
203 num_cpp = osh_native["pass"]
204 if (spec_name == "TOTAL") {
205 href = ""
206 } else {
207 href = sprintf("%s.html", spec_name)
208 }
209
210 if (num_py == num_cpp) {
211 row_css_class = "cpp-good" # green
212 }
213
214 printf("%s,%s,%s,%d,%d,%d\n",
215 row_css_class,
216 spec_name, href,
217 num_py,
218 num_cpp,
219 num_py - num_cpp)
220}
221' "${tsv_files[@]}"
222}
223
224summary-csv() {
225 local spec_subdir=$1
226
227 local sh_label
228 local manifest
229
230 case $spec_subdir in
231 osh-cpp)
232 sh_label=osh
233 manifest=_tmp/spec/SUITE-osh.txt
234 ;;
235 ysh-cpp)
236 sh_label=ysh
237 manifest=_tmp/spec/SUITE-ysh.txt
238 ;;
239 *)
240 die "Invalid dir $spec_subdir"
241 ;;
242 esac
243
244 # Can't go at the top level because files might not exist!
245 cat <<EOF
246ROW_CSS_CLASS,name,name_HREF,${sh_label}_py,${sh_label}_cpp,delta
247EOF
248
249 # total row rows goes at the TOP, so it's in <thead> and not sorted.
250 summary-csv-row $spec_subdir _tmp/spec/$spec_subdir/*.tsv
251
252 head -n $NUM_SPEC_TASKS $manifest | sort |
253 while read spec_name; do
254 summary-csv-row $spec_subdir $spec_name
255 done
256}
257
258html-summary-header() {
259 local prefix=../../..
260 html-head --title 'Passing Spec Tests in C++' \
261 $prefix/web/ajax.js \
262 $prefix/web/table/table-sort.js $prefix/web/table/table-sort.css \
263 $prefix/web/base.css \
264 $prefix/web/spec-cpp.css
265
266 table-sort-begin "width50"
267
268 cat <<EOF
269<p id="home-link">
270 <!-- The release index is two dirs up -->
271 <a href="../..">Up</a> |
272 <a href="/">oilshell.org</a>
273</p>
274
275<h1>Python vs C++</h1>
276
277<p>These numbers measure the progress of the C++ translation.
278Compare with <a href=".">index.html</a>.
279</p>
280
281EOF
282}
283
284html-summary-footer() {
285 cat <<EOF
286<p>Generated by <code>test/spec-cpp.sh</code>.</p>
287EOF
288 table-sort-end 'summary' # The table name
289}
290
291# TODO: Use here-schema-tsv in test/tsv-lib.sh
292here-schema() {
293 ### Read a legible text format on stdin, and write CSV on stdout
294
295 # This is a little like: https://wiki.xxiivv.com/site/tablatal.html
296 # TODO: generalize this in stdlib/here.sh
297 while read one two; do
298 echo "$one,$two"
299 done
300}
301
302write-compare-html() {
303 local spec_subdir=$1
304
305 local sh_label
306 case $spec_subdir in
307 osh-cpp)
308 sh_label=osh
309 ;;
310 ysh-cpp)
311 sh_label=ysh
312 ;;
313 *)
314 die "Invalid dir $spec_subdir"
315 ;;
316 esac
317
318 local dir=_tmp/spec/$spec_subdir
319 local out=$dir/compare.html
320
321 summary-csv $spec_subdir >$dir/summary.csv
322
323 # The underscores are stripped when we don't want them to be!
324 # Note: we could also put "pretty_heading" in the schema
325
326 here-schema >$dir/summary.schema.csv <<EOF
327column_name type
328ROW_CSS_CLASS string
329name string
330name_HREF string
331${sh_label}_py integer
332${sh_label}_cpp integer
333delta integer
334EOF
335
336 { html-summary-header
337 # total row isn't sorted
338 web/table/csv2html.py --thead-offset 1 $dir/summary.csv
339 html-summary-footer
340 } > $out
341
342 log "Comparison: file://$REPO_ROOT/$out"
343}
344
345tsv-demo() {
346 sh-spec spec/arith.test.sh --tsv-output _tmp/arith.tsv dash bash "$@"
347 cat _tmp/arith.tsv
348}
349
350repro() {
351 test/spec.sh alias -r 0 -p > _tmp/a
352 ninja _bin/clang-dbg/osh
353 _bin/clang-dbg/osh _tmp/a
354}
355
356repro-all() {
357 OSH_CC=$REPO_ROOT/_bin/clang-dbg/osh $0 all
358}
359
360task-five "$@"