OILS / core / test_lib.py View on Github | oilshell.org

398 lines, 278 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value
19from asdl import pybase
20from builtin import assign_osh
21from builtin import completion_osh
22from builtin import hay_ysh
23from builtin import io_osh
24from builtin import pure_osh
25from builtin import readline_osh
26from builtin import trap_osh
27from core import alloc
28from core import completion
29from core import dev
30from core import executor
31from core import main_loop
32from core import optview
33from core import process
34from core import pyos
35from core import pyutil
36from core import state
37from core import ui
38from core import util
39from core import vm
40from frontend import lexer
41from frontend import location
42from frontend import parse_lib
43from frontend import reader
44from osh import cmd_eval
45from osh import prompt
46from osh import sh_expr_eval
47from osh import split
48from osh import word_eval
49from ysh import expr_eval
50from mycpp import mylib
51
52import posix_ as posix
53
54
55def MakeBuiltinArgv(argv):
56 return cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, None)
57
58
59def FakeTok(id_, val):
60 # type: (int, str) -> Token
61 src = source.Interactive
62 line = SourceLine(1, val, src)
63 return Token(id_, len(val), 0, line, None)
64
65
66def PrintableString(s):
67 """For pretty-printing in tests."""
68 if all(c in string.printable for c in s):
69 return s
70 return repr(s)
71
72
73def TokensEqual(left, right):
74 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
75
76 if left.id != right.id:
77 return False
78
79 if left.line is not None:
80 left_str = lexer.TokenVal(left)
81 else:
82 left_str = None
83
84 if right.line is not None:
85 right_str = lexer.TokenVal(right)
86 else:
87 right_str = None
88
89 # Better error message sometimes:
90 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
91 return left_str == right_str
92
93
94def TokenWordsEqual(left, right):
95 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
96 return TokensEqual(left.token, right.token)
97 #return left == right
98
99
100def AsdlEqual(left, right):
101 """Check if generated ASDL instances are equal.
102
103 We don't use equality in the actual code, so this is relegated to
104 test_lib.
105 """
106 if left is None and right is None:
107 return True
108
109 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
110 return left == right
111
112 if isinstance(left, list):
113 if len(left) != len(right):
114 return False
115 for a, b in zip(left, right):
116 if not AsdlEqual(a, b):
117 return False
118 return True
119
120 if isinstance(left, pybase.CompoundObj):
121 if left.tag() != right.tag():
122 return False
123
124 field_names = left.__slots__ # hack for now
125 for name in field_names:
126 # Special case: we are not testing locations right now.
127 if name == 'span_id':
128 continue
129 a = getattr(left, name)
130 b = getattr(right, name)
131 if not AsdlEqual(a, b):
132 return False
133
134 return True
135
136 raise AssertionError(left)
137
138
139def AssertAsdlEqual(test, left, right):
140 test.assertTrue(AsdlEqual(left, right),
141 'Expected %s, got %s' % (left, right))
142
143
144def MakeArena(source_name):
145 arena = alloc.Arena(save_tokens=True)
146 arena.PushSource(source.MainFile(source_name))
147 return arena
148
149
150def InitLineLexer(s, arena):
151 line_lexer = lexer.LineLexer(arena)
152 src = source.Interactive
153 line_lexer.Reset(SourceLine(1, s, src), 0)
154 return line_lexer
155
156
157def InitLexer(s, arena):
158 """For tests only."""
159 line_lexer = lexer.LineLexer(arena)
160 line_reader = reader.StringLineReader(s, arena)
161 lx = lexer.Lexer(line_lexer, line_reader)
162 return line_reader, lx
163
164
165def InitWordEvaluator(exec_opts=None):
166 arena = MakeArena('<InitWordEvaluator>')
167 mem = state.Mem('', [], arena, [])
168
169 if exec_opts is None:
170 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
171 mem.exec_opts = exec_opts # circular dep
172 state.InitMem(mem, {}, '0.1')
173 mutable_opts.Init()
174 else:
175 mutable_opts = None
176
177 cmd_deps = cmd_eval.Deps()
178 cmd_deps.trap_nodes = []
179
180 splitter = split.SplitContext(mem)
181 errfmt = ui.ErrorFormatter()
182
183 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
184 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
185 tilde_ev, splitter, errfmt)
186 return ev
187
188
189def InitCommandEvaluator(parse_ctx=None,
190 comp_lookup=None,
191 arena=None,
192 mem=None,
193 aliases=None,
194 ext_prog=None):
195
196 opt0_array = state.InitOpts()
197 opt_stacks = [None] * option_i.ARRAY_SIZE
198 if parse_ctx:
199 arena = parse_ctx.arena
200 else:
201 parse_ctx = InitParseContext()
202
203 mem = mem or state.Mem('', [], arena, [])
204 exec_opts = optview.Exec(opt0_array, opt_stacks)
205 mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
206 mem.exec_opts = exec_opts
207 state.InitMem(mem, {}, '0.1')
208 mutable_opts.Init()
209
210 # No 'readline' in the tests.
211
212 errfmt = ui.ErrorFormatter()
213 job_control = process.JobControl()
214 job_list = process.JobList()
215 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None)
216 aliases = {} if aliases is None else aliases
217 procs = {}
218 methods = {}
219
220 compopt_state = completion.OptionState()
221 comp_lookup = comp_lookup or completion.Lookup()
222
223 readline = None # simulate not having it
224
225 new_var = assign_osh.NewVar(mem, procs, errfmt)
226 assign_builtins = {
227 builtin_i.declare: new_var,
228 builtin_i.typeset: new_var,
229 builtin_i.local: new_var,
230 builtin_i.export_: assign_osh.Export(mem, errfmt),
231 builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
232 }
233 builtins = { # Lookup
234 builtin_i.echo: io_osh.Echo(exec_opts),
235 builtin_i.shift: assign_osh.Shift(mem),
236
237 builtin_i.history: readline_osh.History(
238 readline,
239 mem,
240 errfmt,
241 mylib.Stdout(),
242 ),
243
244 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
245 builtin_i.compadjust: completion_osh.CompAdjust(mem),
246
247 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
248 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
249 }
250
251 debug_f = util.DebugFile(sys.stderr)
252 cmd_deps = cmd_eval.Deps()
253 cmd_deps.mutable_opts = mutable_opts
254
255 search_path = state.SearchPath(mem)
256
257 ext_prog = \
258 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
259
260 cmd_deps.dumper = dev.CrashDumper('', fd_state)
261 cmd_deps.debug_f = debug_f
262
263 splitter = split.SplitContext(mem)
264
265 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
266 parse_ctx, errfmt)
267 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
268 parse_ctx, errfmt)
269 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
270 errfmt)
271 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
272 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
273 tilde_ev, splitter, errfmt)
274 signal_safe = pyos.InitSignalSafe()
275 trap_state = trap_osh.TrapState(signal_safe)
276 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
277 assign_builtins, arena, cmd_deps,
278 trap_state, signal_safe)
279
280 multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
281 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
282 multi_trace)
283 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
284
285 hay_state = hay_ysh.HayState()
286 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
287 hay_state, builtins, search_path,
288 ext_prog, waiter, tracer, job_control,
289 job_list, fd_state, trap_state, errfmt)
290
291 assert cmd_ev.mutable_opts is not None, cmd_ev
292 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
293
294 global_io = value.IO(cmd_ev, prompt_ev)
295 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
296 prompt_ev, global_io, tracer)
297
298 try:
299 from _devbuild.gen.help_meta import TOPICS
300 except ImportError:
301 TOPICS = None # minimal dev build
302 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
303 splitter, comp_lookup, TOPICS,
304 errfmt)
305
306 # Add some builtins that depend on the executor!
307 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
308 builtins[builtin_i.complete] = complete_builtin
309 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
310
311 return cmd_ev
312
313
314def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
315 """Unit tests can evaluate code strings and then use the resulting
316 CommandEvaluator."""
317 arena = parse_ctx.arena
318 errfmt = ui.ErrorFormatter()
319
320 comp_lookup = comp_lookup or completion.Lookup()
321 mem = mem or state.Mem('', [], arena, [])
322 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
323 mem.exec_opts = exec_opts
324
325 state.InitMem(mem, {}, '0.1')
326 mutable_opts.Init()
327
328 line_reader, _ = InitLexer(code_str, arena)
329 c_parser = parse_ctx.MakeOshParser(line_reader)
330
331 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
332 comp_lookup=comp_lookup,
333 arena=arena,
334 mem=mem,
335 aliases=aliases)
336
337 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
338 return cmd_ev
339
340
341def InitParseContext(arena=None,
342 ysh_grammar=None,
343 aliases=None,
344 parse_opts=None,
345 do_lossless=False):
346 arena = arena or MakeArena('<test_lib>')
347
348 if aliases is None:
349 aliases = {}
350
351 mem = state.Mem('', [], arena, [])
352 if parse_opts is None:
353 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
354
355 parse_ctx = parse_lib.ParseContext(arena,
356 parse_opts,
357 aliases,
358 ysh_grammar,
359 do_lossless=do_lossless)
360
361 return parse_ctx
362
363
364def InitWordParser(word_str, oil_at=False, arena=None):
365 arena = arena or MakeArena('<test_lib>')
366
367 mem = state.Mem('', [], arena, [])
368 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
369
370 # CUSTOM SETTING
371 mutable_opts.opt0_array[option_i.parse_at] = oil_at
372
373 loader = pyutil.GetResourceLoader()
374 ysh_grammar = pyutil.LoadYshGrammar(loader)
375 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
376 line_reader, _ = InitLexer(word_str, arena)
377 c_parser = parse_ctx.MakeOshParser(line_reader)
378 # Hack
379 return c_parser.w_parser
380
381
382def InitCommandParser(code_str, arena=None):
383 arena = arena or MakeArena('<test_lib>')
384
385 loader = pyutil.GetResourceLoader()
386 ysh_grammar = pyutil.LoadYshGrammar(loader)
387
388 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
389 line_reader, _ = InitLexer(code_str, arena)
390 c_parser = parse_ctx.MakeOshParser(line_reader)
391 return c_parser
392
393
394def SetLocalString(mem, name, s):
395 # type: (state.Mem, str, str) -> None
396 """Bind a local string."""
397 assert isinstance(s, str)
398 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)