OILS / mycpp / TEST.sh View on Github | oilshell.org

339 lines, 209 significant
1#!/usr/bin/env bash
2#
3# Run tests in this directory.
4#
5# Usage:
6# mycpp/TEST.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11
12REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
13source build/common.sh
14source build/ninja-rules-cpp.sh
15source devtools/common.sh
16source soil/common.sh # find-dir-html
17source test/common.sh # run-test-bin, can-compile-32-bit
18
19# in case binaries weren't built
20shopt -s failglob
21
22# Will be needed to pass ASAN leak detector? Or only do this for the main binary?
23# export OILS_GC_ON_EXIT=1
24
25examples-variant() {
26 ### Run all examples using a variant -- STATS only
27
28 local compiler=${1:-cxx}
29 local variant=${2:-asan+gcalways}
30 local do_benchmark=${3:-}
31
32 banner "$0 examples-variant $compiler $variant"
33
34 ninja mycpp-examples-$compiler-$variant
35
36 local num_tests=0
37 local num_failed=0
38 local status=0
39
40 local log_dir=_test/$compiler-$variant/mycpp/examples
41 mkdir -p $log_dir
42
43 for b in _bin/$compiler-$variant/mycpp/examples/*; do
44 case $b in
45 (*.stripped) # just run the unstripped binary
46 continue
47 ;;
48 esac
49
50 local prefix="$log_dir/$(basename $b)"
51
52 case $variant in
53 (coverage)
54 export LLVM_PROFILE_FILE=$prefix.profraw
55 ;;
56 esac
57
58 local log="${prefix}${do_benchmark}.log"
59
60 log "RUN $b > $log"
61
62 local test_name=$(basename $b)
63 if test -n "$do_benchmark" && [[ $test_name == test_* ]]; then
64 log "Skipping $test_name in benchmark mode"
65 continue
66 fi
67
68 set +o errexit
69 BENCHMARK="$do_benchmark" $b >$log 2>&1
70 status=$?
71 set -o errexit
72
73 if test "$status" -eq 0; then
74 log 'OK'
75 else
76 log "FAIL with status $?"
77 log ''
78 #return $status
79 num_failed=$((num_failed + 1))
80 fi
81
82 num_tests=$((num_tests + 1))
83 done
84
85 log ''
86 log "$num_failed of $num_tests tests failed"
87 log ''
88
89 if test $num_failed -ne 0; then
90 echo "FAIL: Expected no failures, got $num_failed"
91 return 1
92 fi
93
94 return 0
95}
96
97#
98# 3 Variants x {test, benchmark}
99#
100
101ex-gcalways() {
102 local compiler=${1:-}
103 examples-variant "$compiler" asan+gcalways
104}
105
106# TOO SLOW to run. It's garbage collecting all the time.
107ex-gcalways-bench() {
108 local compiler=${1:-}
109 examples-variant "$compiler" asan+gcalways '.BENCHMARK'
110}
111
112ex-asan() {
113 local compiler=${1:-}
114 examples-variant "$compiler" asan
115}
116
117# 2 of 18 tests failed: cartesian, parse
118# So it does not catch the 10 segfaults that 'asan+gcalways' catches with a few
119# iterations!
120ex-asan-bench() {
121 local compiler=${1:-}
122 examples-variant "$compiler" asan '.BENCHMARK'
123}
124
125# PASS! Under both clang and GCC.
126ex-ubsan() {
127 local compiler=${1:-}
128 examples-variant "$compiler" ubsan
129}
130
131# same as ASAN: 2 of 18
132ex-ubsan-bench() {
133 local compiler=${1:-}
134 examples-variant "$compiler" ubsan '.BENCHMARK'
135}
136
137# PASS!
138ex-opt() {
139 local compiler=${1:-}
140 examples-variant "$compiler" opt
141}
142
143# 2 of 18 tests failed
144ex-opt-bench() {
145 local compiler=${1:-}
146 examples-variant "$compiler" opt '.BENCHMARK'
147}
148
149#
150# Unit Tests
151#
152
153unit() {
154 ### Run by test/cpp-unit.sh
155
156 local compiler=${1:-cxx}
157 local variant=${2:-asan+gcalways}
158
159 log ''
160 log "$0 unit $compiler $variant"
161 log ''
162
163 ninja mycpp-unit-$compiler-$variant
164
165 local -a binaries=(_bin/$compiler-$variant/mycpp/*)
166
167 # Add these files if they exist in the variant
168 if test -d _bin/$compiler-$variant/mycpp/demo; then
169 binaries+=(_bin/$compiler-$variant/mycpp/demo/*)
170 fi
171
172 for b in "${binaries[@]}"; do
173 if ! test -f $b; then
174 continue
175 fi
176
177 local prefix=${b//_bin/_test/}
178 local log=$prefix.log
179 mkdir -p $(dirname $log)
180
181 local asan_options=''
182 case $b in
183 # leaks with malloc
184 (*/demo/hash_table|*/demo/target_lang|*/demo/gc_header|*/small_str_test)
185 asan_options='detect_leaks=0'
186 ;;
187 esac
188
189 ASAN_OPTIONS="$asan_options" run-test-bin $b
190
191 done
192}
193
194#
195# Test failures
196#
197
198translate-example() {
199 local ex=$1
200
201 local mycpp=_bin/shwrap/mycpp_main
202 $mycpp '.:pyext' _tmp/mycpp-invalid $ex
203}
204
205test-invalid-examples() {
206 local mycpp=_bin/shwrap/mycpp_main
207 ninja $mycpp
208 for ex in mycpp/examples/invalid_*; do
209
210 banner "$ex"
211
212 set +o errexit
213 translate-example $ex
214 local status=$?
215 set -o errexit
216
217 local expected_status=1
218
219 case $ex in
220 */invalid_condition.py)
221 expected_status=8
222 ;;
223 */invalid_default_args.py)
224 expected_status=4
225 ;;
226 */invalid_try_else.py)
227 expected_status=3
228 ;;
229 */invalid_except.py)
230 expected_status=3
231 ;;
232 */invalid_global.py)
233 expected_status=2
234 ;;
235 */invalid_switch.py)
236 expected_status=5
237 ;;
238 esac
239
240 if test $status -ne $expected_status; then
241 die "mycpp $ex: expected status $expected_status, got $status"
242 fi
243
244 done
245}
246
247test-runtime() {
248 # Run other unit tests, e.g. the GC tests
249
250 # Special test
251
252 # TODO: Switch when the CI supports it
253 # We also want to change test/cpp-unit.sh all-tests
254
255 local ubsan_compiler=cxx
256 #local ubsan_compiler=clang
257
258 for config in cxx-asan+bumpleak $ubsan_compiler-ubsan+bumpleak; do
259 local bin=_bin/$config/mycpp/bump_leak_heap_test
260 ninja $bin
261 run-test-bin $bin
262 done
263
264 # Run other tests with all variants
265
266 unit $ubsan_compiler ubsan
267
268 unit '' asan
269 unit '' asan+gcalways
270 unit '' opt
271
272 if can-compile-32-bit; then
273 unit '' asan32+gcalways # ASAN on 32-bit
274 else
275 log "Can't compile 32-bit binaries (gcc-multilib g++-multilib needed on Debian)"
276 fi
277}
278
279#
280# Translator
281#
282
283test-translator() {
284 ### Invoked by soil/worker.sh
285
286 examples-variant '' asan
287
288 # Test with more collections
289 examples-variant '' asan+gcalways
290
291 run-test-func test-invalid-examples _test/mycpp/test-invalid-examples.log
292
293 # Runs tests in cxx-asan variant, and benchmarks in cxx-opt variant
294 if ! ninja mycpp-logs-equal; then
295 log 'FAIL mycpp-logs-equal'
296 return 1
297 fi
298}
299
300soil-run() {
301 set +o errexit
302 $0 test-translator
303 local status=$?
304 set -o errexit
305
306 # Write _test/mycpp-examples.html, used by soil/woker.sh
307 find-dir-html _test mycpp-examples
308
309 return $status
310}
311
312unit-test-coverage() {
313 ### Invoked by Soil
314
315 local bin=_bin/clang-coverage+bumpleak/mycpp/bump_leak_heap_test
316 ninja $bin
317 run-test-bin $bin
318
319 unit clang coverage
320
321 local out_dir=_test/clang-coverage/mycpp
322 test/coverage.sh html-report $out_dir \
323 clang-coverage/mycpp clang-coverage+bumpleak/mycpp
324}
325
326examples-coverage() {
327 ### Invoked by Soil
328
329 examples-variant clang coverage
330
331 local out_dir=_test/clang-coverage/mycpp/examples
332 test/coverage.sh html-report $out_dir clang-coverage/mycpp/examples
333}
334
335# Call function $1 with arguments $2 $3 $4
336#
337# mycpp/TEST.sh examples-variant '' asan
338
339"$@"