OILS / test / stateful.sh View on Github | oilshell.org

232 lines, 98 significant
1#!/usr/bin/env bash
2#
3# Wrapper for test cases in spec/stateful
4#
5# Usage:
6# test/stateful.sh <function name>
7#
8# Examples:
9# test/stateful.sh signals -r 0-1 # run a range of tests
10# test/stateful.sh signals --list # list tests
11# test/stateful.sh job-control --num-retries 0
12#
13# test/stateful.sh signals-quick # not all shells
14#
15# test/stateful.sh soil-run
16#
17# TODO: Should have QUICKLY=1 variants
18
19set -o nounset
20set -o pipefail
21set -o errexit
22
23REPO_ROOT=$(cd $(dirname $0)/.. && pwd) # tsv-lib.sh uses this
24readonly REPO_ROOT
25
26source test/common.sh # log, $OSH
27source test/tsv-lib.sh
28
29# This uses ../oil_DEPS/spec-bin/{bash,dash} if they exist
30# The ovm-tarball container that has spec-bin doesn't have python3 :-( Really
31# we should build another container
32source build/dev-shell.sh
33
34readonly BASE_DIR=_tmp/spec/stateful
35
36# Hack for testing the harness
37#readonly FIRST='-r 0'
38readonly FIRST=''
39readonly OSH_CPP=_bin/cxx-asan/osh
40
41#readonly -a QUICK_SHELLS=( $OSH bash )
42readonly -a QUICK_SHELLS=( $OSH $OSH_CPP bash )
43
44#
45# Suites in spec/stateful
46#
47
48signals() {
49 spec/stateful/signals.py $FIRST "$@"
50}
51
52interactive() {
53 spec/stateful/interactive.py $FIRST "$@"
54}
55
56job-control() {
57 spec/stateful/job_control.py $FIRST --oils-failures-allowed 0 "$@"
58}
59
60# Run on just 2 shells
61
62signals-quick() { signals "${QUICK_SHELLS[@]}" "$@"; }
63interactive-quick() { interactive "${QUICK_SHELLS[@]}" "$@"; }
64job-control-quick() { job-control "${QUICK_SHELLS[@]}" "$@"; }
65
66# Run on all shells we can
67
68# They now pass for dash and mksh, with wait -n and PIPESTATUS skipped. zsh
69# doesn't work now, but could if the prompt was changed to $ ?
70signals-all() { signals "${QUICK_SHELLS[@]}" dash mksh "$@"; }
71
72interactive-all() { interactive "${QUICK_SHELLS[@]}" dash mksh "$@"; }
73
74job-control-all() { job-control "${QUICK_SHELLS[@]}" dash "$@"; }
75
76#
77# More automation
78#
79
80print-tasks() {
81 ### List all tests
82
83 # TODO:
84 # - Print a table with --osh-allowed-failures and shells. It can be filtered
85
86 if test -n "${QUICKLY:-}"; then
87 echo 'interactive'
88 else
89 echo 'interactive'
90 echo 'job-control'
91 echo 'signals'
92 fi
93}
94
95run-file() {
96 ### Run a spec/stateful file, logging output
97
98 local spec_name=$1
99
100 log "__ $spec_name"
101
102 local base_dir=$BASE_DIR
103
104 local log_filename=$spec_name.log.txt
105 local results_filename=$spec_name.results.txt
106
107 time-tsv -o $base_dir/${spec_name}.task.txt \
108 --field $spec_name --field $log_filename --field $results_filename -- \
109 $0 "$spec_name-all" --results-file $base_dir/$results_filename \
110 >$base_dir/$log_filename 2>&1 || true
111}
112
113html-summary() {
114 ### Summarize all files
115
116 # Note: In retrospect, it would be better if every process writes a "long"
117 # TSV file of results.
118 # And then we concatenate them and write the "wide" summary here.
119
120 html-head --title 'Stateful Tests' \
121 ../../../web/base.css ../../../web/spec-tests.css
122
123 # Similar to test/spec-runner.sh and soil format-wwz-index
124
125 cat <<EOF
126 <body class="width50">
127
128<p id="home-link">
129 <!-- up to .wwz index -->
130 <a href="../..">Up</a> |
131 <a href="/">Home</a>
132</p>
133
134 <h1>Stateful Tests with <a href="//www.oilshell.org/cross-ref.html#pexpect">pexpect</a> </h1>
135
136 <table>
137 <thead>
138 <tr>
139 <td>Test File</td>
140 <td>Elapsed seconds</td>
141 <td>Status</td>
142 </tr>
143 </thead>
144EOF
145
146 local all_passed=0
147
148 shopt -s lastpipe # to mutate all_passed in while
149
150 local results_tmp=$BASE_DIR/results.html
151 echo '' > $results_tmp # Accumulate more here
152
153 print-tasks | while read spec_name; do
154
155 # Note: in test/spec-runner.sh, an awk script creates this table. It reads
156 # *.task.txt and *.stats.txt. I could add --stats-file to harness.py
157 # with pass/fail stats
158 read status elapsed _ log_filename results_filename < $BASE_DIR/${spec_name}.task.txt
159
160 echo '<tr>'
161 echo "<td> <a href="$log_filename">$spec_name</a> </td>"
162
163 printf -v elapsed_str '%.1f' $elapsed
164 echo "<td>$elapsed_str</td>"
165
166 case $status in
167 (0) # exit code 0 is success
168 echo " <td>$status</td>"
169 ;;
170 (*) # everything else is a failure
171 # Add extra text to make red stand out.
172 echo " <td class=\"fail\">status: $status</td>"
173
174 # Mark failure
175 all_passed=1
176 ;;
177 esac
178 echo '</tr>'
179
180 # Append to temp file
181 {
182 echo "<h2>$spec_name</h2>"
183 echo '<pre>'
184 escape-html $BASE_DIR/$results_filename
185 echo '</pre>'
186 } >> $results_tmp
187
188 done
189 echo '</table>'
190
191 cat $results_tmp
192
193 cat <<EOF
194 </table>
195 </body>
196</html>
197EOF
198
199 log "all_passed = $all_passed"
200
201 return $all_passed
202}
203
204soil-run() {
205 ninja $OSH_CPP
206
207 mkdir -p $BASE_DIR
208
209 print-tasks | xargs -n 1 -- $0 run-file
210
211 # Returns whether all passed
212 html-summary > $BASE_DIR/index.html
213}
214
215#
216# Debugging
217#
218
219test-stop() {
220 python3 spec/stateful/harness.py test-stop demo/cpython/fork_signal_state.py
221}
222
223strace-py-fork() {
224 rm -f -v _tmp/py-fork.*
225 strace -ff -o _tmp/py-fork demo/cpython/fork_signal_state.py
226 ls -l _tmp/py-fork.*
227
228 # I see rt_sigaction(SIGSTP, ...) which is good
229 # so yeah this seems perfectly fine -- why is it ignoring SIGTSTP? :-(
230}
231
232"$@"