OILS / test / lint.sh View on Github | oilshell.org

324 lines, 180 significant
1#!/usr/bin/env bash
2#
3# Run tools to maintain the coding style.
4#
5# Usage:
6# test/lint.sh <function name>
7
8set -o nounset
9set -o pipefail
10set -o errexit
11shopt -s strict:all 2>/dev/null || true # dogfood for OSH
12
13REPO_ROOT=$(cd "$(dirname $0)/.."; pwd)
14readonly REPO_ROOT
15
16source build/common.sh
17source build/dev-shell.sh # python2 and python3
18source devtools/common.sh # banner
19source devtools/run-task.sh # run-task
20
21# TODO: synchronize with metrics/source-code.sh
22readonly -a CODE_DIRS=(asdl bin builtin core data_lang frontend osh tools yaks ysh)
23
24#
25# C++
26#
27
28get-cpplint() {
29 mkdir -p _tmp
30 wget --directory _tmp \
31 https://raw.githubusercontent.com/google/styleguide/gh-pages/cpplint/cpplint.py
32 chmod +x _tmp/cpplint.py
33}
34
35cpplint() {
36 # we don't have subdir names on the header guard
37 _tmp/cpplint.py --filter \
38 -readability/todo,-legal/copyright,-build/header_guard,-build/include,-whitespace/comments "$@"
39}
40
41#
42# Python
43#
44
45find-prune() {
46 ### find real source files
47
48 # benchmarks/testdata should be excluded
49 # excluding _build, _devbuild. Although it might be OK to test generated
50 # code for tabs.
51 find . '(' -type d -a -name '_*' \
52 -o -name testdata \
53 -o -name $PY27 \
54 ')' -a -prune \
55 "$@"
56}
57
58find-src-files() {
59 find-prune \
60 -o -type f -a \
61 '(' -name '*.py' \
62 -o -name '*.sh' \
63 -o -name '*.asdl' \
64 -o -name '*.[ch]' \
65 -o -name '*.cc' \
66 ')' -a -print
67}
68
69find-tabs() {
70 find-src-files | xargs grep -n $'\t'
71}
72
73find-long-lines() {
74 # Exclude URLs
75 find-src-files | xargs grep -n '^.\{81\}' | grep -v 'http'
76}
77
78#
79# pyflakes-based lint
80#
81
82oils-lint() {
83 local lang=$1 # py2 or py3
84 shift
85
86 PYTHONPATH=.:~/wedge/oils-for-unix.org/pkg/pyflakes/2.4.0 test/${lang}_lint.py "$@"
87 #PYTHONPATH=.:vendor/pyflakes-2.4.0 test/oils_lint.py "$@"
88}
89
90py2-lint() {
91 oils-lint py2 "$@"
92}
93
94py3-lint() {
95 oils-lint py3 "$@"
96}
97
98py2() {
99 banner 'Linting Python 2 code'
100
101 # syntax_abbrev.py doesn't stand alone
102 py2-files-to-format | grep -v '_abbrev.py' | xargs $0 py2-lint
103}
104
105py3-files() {
106 for f in mycpp/*.py; do
107 echo $f
108 done
109}
110
111py3() {
112 banner 'Linting Python 3 code'
113
114 py3-files | xargs $0 py3-lint
115}
116
117all-py() {
118 py2
119 py3
120}
121
122#
123# More Python, including Python 3
124#
125
126mycpp-files() {
127 for f in mycpp/*.py; do
128 case $f in
129 */NINJA_subgraph.py)
130 continue
131 ;;
132 esac
133
134 echo $f
135 done
136}
137
138py2-files-to-format() {
139 for dir in "${CODE_DIRS[@]}"; do
140 for name in $dir/*.py; do
141 echo $name
142 done
143 done | grep -v 'NINJA_subgraph' # leave out for now
144}
145
146run-docformatter() {
147 ### Format docstrings
148
149 # Only done as a ONE OFF to indent docstrings after yapf-2
150 # Because it tends to mangle comments, e.g. grammar comments in
151 # ysh/expr_to_ast.py
152 time py2-files-to-format \
153 | xargs --verbose -- python3 -m docformatter --in-place
154}
155
156#
157# Main
158#
159
160# Hook for soil
161soil-run() {
162 if test -n "${TRAVIS_SKIP:-}"; then
163 echo "TRAVIS_SKIP: Skipping $0"
164 return
165 fi
166
167 #flake8-all
168
169 # Our new lint script
170 all-py
171
172 check-shebangs
173}
174
175#
176# Adjust and Check shebang lines. It matters for developers on different distros.
177#
178
179find-files-to-lint() {
180 ### Similar to find-prune / find-src-files
181
182 # don't touch mycpp yet because it's in Python 3
183 # build has build/dynamic_deps.py which needs the -S
184 find . \
185 -name '_*' -a -prune -o \
186 -name 'Python-*' -a -prune -o \
187 "$@"
188}
189
190find-py() {
191 find-files-to-lint \
192 -name 'build' -a -prune -o \
193 -name '*.py' -a -print "$@"
194}
195
196find-sh() {
197 find-files-to-lint -name '*.sh' -a -print "$@"
198}
199
200print-if-has-shebang() {
201 read first < $1
202 [[ "$first" == '#!'* ]] && echo $1
203}
204
205not-executable() {
206 find-py -a ! -executable -a -print | xargs -n 1 -- $0 print-if-has-shebang
207}
208
209executable-py() {
210 find-py -a -executable -a -print | xargs -n 1 -- echo
211}
212
213# Make all shebangs consistent.
214# - Specify python2 because on some distros 'python' is python3
215# - Use /usr/bin/env because it works better with virtualenv?
216#
217# https://stackoverflow.com/questions/9309940/sed-replace-first-line
218#
219# e.g. cat edit.list, change the first line
220
221replace-py-shebang() {
222 sed -i '1c#!/usr/bin/env python2' "$@"
223}
224
225replace-bash-shebang() {
226 sed -i '1c#!/usr/bin/env bash' "$@"
227}
228
229# NOTE: no ^ anchor because of print-first-line
230
231readonly BAD_PY='#!.*/usr/bin/python'
232readonly BAD_BASH='#!.*/bin/bash'
233
234bad-py() {
235 find-py -a -print | xargs -- egrep "$BAD_PY"
236 #grep '^#!.*/bin/bash ' */*.sh
237
238 find-py -a -print | xargs -- egrep -l "$BAD_PY" | xargs $0 replace-py-shebang
239}
240
241bad-bash() {
242 # these files don't need shebangs
243 #grep -l '^#!' spec/*.test.sh | xargs -- sed -i '1d'
244
245 #find-sh -a -print | xargs -- grep "$BAD_BASH"
246
247 find-sh -a -print | xargs -- egrep -l "$BAD_BASH" | xargs $0 replace-bash-shebang
248}
249
250print-first-line() {
251 local path=$1
252
253 read line < "$path"
254 echo "$path: $line" # like grep output
255}
256
257check-shebangs() {
258 set +o errexit
259
260 if true; then
261 find-py | xargs -d $'\n' -n 1 -- $0 print-first-line | egrep "$BAD_PY"
262 if test $? -ne 1; then
263 die "FAIL: Found bad Python shebangs"
264 fi
265 fi
266
267 find-sh | xargs -d $'\n' -n 1 -- $0 print-first-line | egrep "$BAD_BASH"
268 if test $? -ne 1; then
269 die "FAIL: Found bad bash shebangs"
270 fi
271
272 echo 'PASS: check-shebangs'
273}
274
275#
276# sprintf -- What do we need in mycpp?
277#
278
279sp-formats() {
280 egrep --no-filename --only-matching '%.' */*.py | sort | uniq -c | sort -n
281}
282
283# 122 instances of these. %() for named
284sp-rare() {
285 egrep --color=always '%[^srd ]' */*.py | egrep -v 'Python-|_test.py'
286}
287
288#
289# inherit
290#
291
292# 56 instances of inheritance
293inheritance() {
294 grep ^class {osh,core,ysh,frontend}/*.py \
295 | egrep -v '_test|object'
296}
297
298# 18 unique base classes.
299# TODO: Maybe extract this automatically with OPy?
300# Or does the MyPy AST have enough?
301# You can collect method defs in the decl phase. Or in the forward_decl phase?
302
303base-classes() {
304 inheritance | egrep -o '\(.*\)' | sort | uniq -c | sort -n
305}
306
307translation() {
308 set +o errexit
309
310 metrics/source-code.sh osh-files \
311 | xargs egrep -n 'IndexError|KeyError'
312 local status=$?
313
314 echo
315
316 # 4 occurrences
317 # source builtin, core/process.py, etc.
318
319 metrics/source-code.sh osh-files \
320 | xargs egrep -n 'finally:'
321 #| xargs egrep -n -A 1 'finally:'
322}
323
324run-task "$@"