OILS / core / test_lib.py View on Github | oilshell.org

395 lines, 276 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value
19from asdl import pybase
20from asdl import runtime
21from builtin import assign_osh
22from builtin import completion_osh
23from builtin import hay_ysh
24from builtin import io_osh
25from builtin import pure_osh
26from builtin import readline_osh
27from builtin import trap_osh
28from core import alloc
29from core import completion
30from core import dev
31from core import executor
32from core import main_loop
33from core import optview
34from core import process
35from core import pyos
36from core import pyutil
37from core import state
38from core import ui
39from core import util
40from core import vm
41from frontend import lexer
42from frontend import location
43from frontend import parse_lib
44from frontend import reader
45from osh import cmd_eval
46from osh import prompt
47from osh import sh_expr_eval
48from osh import split
49from osh import word_eval
50from ysh import expr_eval
51from mycpp import mylib
52
53
54def MakeBuiltinArgv(argv):
55 return cmd_value.Argv(argv, [loc.Missing] * len(argv), None, None, None)
56
57
58def FakeTok(id_, val):
59 # type: (int, str) -> Token
60 src = source.Interactive
61 line = SourceLine(1, val, src)
62 return Token(id_, 0, len(val), runtime.NO_SPID, line, val)
63
64
65def PrintableString(s):
66 """For pretty-printing in tests."""
67 if all(c in string.printable for c in s):
68 return s
69 return repr(s)
70
71
72def TokensEqual(left, right):
73 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
74
75 if left.id != right.id:
76 return False
77
78 if left.line is not None:
79 left_str = lexer.TokenVal(left)
80 else:
81 left_str = None
82
83 if right.line is not None:
84 right_str = lexer.TokenVal(right)
85 else:
86 right_str = None
87
88 # Better error message sometimes:
89 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
90 return left_str == right_str
91
92
93def TokenWordsEqual(left, right):
94 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
95 return TokensEqual(left.token, right.token)
96 #return left == right
97
98
99def AsdlEqual(left, right):
100 """Check if generated ASDL instances are equal.
101
102 We don't use equality in the actual code, so this is relegated to
103 test_lib.
104 """
105 if left is None and right is None:
106 return True
107
108 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
109 return left == right
110
111 if isinstance(left, list):
112 if len(left) != len(right):
113 return False
114 for a, b in zip(left, right):
115 if not AsdlEqual(a, b):
116 return False
117 return True
118
119 if isinstance(left, pybase.CompoundObj):
120 if left.tag() != right.tag():
121 return False
122
123 field_names = left.__slots__ # hack for now
124 for name in field_names:
125 # Special case: we are not testing locations right now.
126 if name == 'span_id':
127 continue
128 a = getattr(left, name)
129 b = getattr(right, name)
130 if not AsdlEqual(a, b):
131 return False
132
133 return True
134
135 raise AssertionError(left)
136
137
138def AssertAsdlEqual(test, left, right):
139 test.assertTrue(AsdlEqual(left, right),
140 'Expected %s, got %s' % (left, right))
141
142
143def MakeArena(source_name):
144 arena = alloc.Arena(save_tokens=True)
145 arena.PushSource(source.MainFile(source_name))
146 return arena
147
148
149def InitLineLexer(s, arena):
150 line_lexer = lexer.LineLexer(arena)
151 src = source.Interactive
152 line_lexer.Reset(SourceLine(1, s, src), 0)
153 return line_lexer
154
155
156def InitLexer(s, arena):
157 """For tests only."""
158 line_lexer = lexer.LineLexer(arena)
159 line_reader = reader.StringLineReader(s, arena)
160 lx = lexer.Lexer(line_lexer, line_reader)
161 return line_reader, lx
162
163
164def InitWordEvaluator(exec_opts=None):
165 arena = MakeArena('<InitWordEvaluator>')
166 mem = state.Mem('', [], arena, [])
167
168 if exec_opts is None:
169 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
170 mem.exec_opts = exec_opts # circular dep
171 state.InitMem(mem, {}, '0.1')
172 mutable_opts.Init()
173 else:
174 mutable_opts = None
175
176 cmd_deps = cmd_eval.Deps()
177 cmd_deps.trap_nodes = []
178
179 splitter = split.SplitContext(mem)
180 errfmt = ui.ErrorFormatter()
181
182 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
183 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
184 tilde_ev, splitter, errfmt)
185 return ev
186
187
188def InitCommandEvaluator(parse_ctx=None,
189 comp_lookup=None,
190 arena=None,
191 mem=None,
192 aliases=None,
193 ext_prog=None):
194
195 opt0_array = state.InitOpts()
196 opt_stacks = [None] * option_i.ARRAY_SIZE
197 if parse_ctx:
198 arena = parse_ctx.arena
199 else:
200 parse_ctx = InitParseContext()
201
202 mem = mem or state.Mem('', [], arena, [])
203 exec_opts = optview.Exec(opt0_array, opt_stacks)
204 mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
205 mem.exec_opts = exec_opts
206 state.InitMem(mem, {}, '0.1')
207 mutable_opts.Init()
208
209 # No 'readline' in the tests.
210
211 errfmt = ui.ErrorFormatter()
212 job_control = process.JobControl()
213 job_list = process.JobList()
214 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None)
215 aliases = {} if aliases is None else aliases
216 procs = {}
217 methods = {}
218
219 compopt_state = completion.OptionState()
220 comp_lookup = comp_lookup or completion.Lookup()
221
222 readline = None # simulate not having it
223
224 new_var = assign_osh.NewVar(mem, procs, errfmt)
225 assign_builtins = {
226 builtin_i.declare: new_var,
227 builtin_i.typeset: new_var,
228 builtin_i.local: new_var,
229 builtin_i.export_: assign_osh.Export(mem, errfmt),
230 builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
231 }
232 builtins = { # Lookup
233 builtin_i.echo: io_osh.Echo(exec_opts),
234 builtin_i.shift: assign_osh.Shift(mem),
235
236 builtin_i.history: readline_osh.History(
237 readline,
238 mem,
239 errfmt,
240 mylib.Stdout(),
241 ),
242
243 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
244 builtin_i.compadjust: completion_osh.CompAdjust(mem),
245
246 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
247 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
248 }
249
250 debug_f = util.DebugFile(sys.stderr)
251 cmd_deps = cmd_eval.Deps()
252 cmd_deps.mutable_opts = mutable_opts
253
254 search_path = state.SearchPath(mem)
255
256 ext_prog = \
257 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
258
259 cmd_deps.dumper = dev.CrashDumper('', fd_state)
260 cmd_deps.debug_f = debug_f
261
262 splitter = split.SplitContext(mem)
263
264 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
265 parse_ctx, errfmt)
266 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
267 parse_ctx, errfmt)
268 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
269 errfmt)
270 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
271 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
272 tilde_ev, splitter, errfmt)
273 signal_safe = pyos.InitSignalSafe()
274 trap_state = trap_osh.TrapState(signal_safe)
275 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
276 assign_builtins, arena, cmd_deps,
277 trap_state, signal_safe)
278
279 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f)
280 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
281
282 hay_state = hay_ysh.HayState()
283 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
284 hay_state, builtins, search_path,
285 ext_prog, waiter, tracer, job_control,
286 job_list, fd_state, trap_state, errfmt)
287
288 assert cmd_ev.mutable_opts is not None, cmd_ev
289 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
290
291 global_io = value.IO(cmd_ev, prompt_ev)
292 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
293 prompt_ev, global_io, tracer)
294
295 try:
296 from _devbuild.gen.help_meta import TOPICS
297 except ImportError:
298 TOPICS = None # minimal dev build
299 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
300 splitter, comp_lookup, TOPICS,
301 errfmt)
302
303 # Add some builtins that depend on the executor!
304 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
305 builtins[builtin_i.complete] = complete_builtin
306 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
307
308 return cmd_ev
309
310
311def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
312 """Unit tests can evaluate code strings and then use the resulting
313 CommandEvaluator."""
314 arena = parse_ctx.arena
315 errfmt = ui.ErrorFormatter()
316
317 comp_lookup = comp_lookup or completion.Lookup()
318 mem = mem or state.Mem('', [], arena, [])
319 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
320 mem.exec_opts = exec_opts
321
322 state.InitMem(mem, {}, '0.1')
323 mutable_opts.Init()
324
325 line_reader, _ = InitLexer(code_str, arena)
326 c_parser = parse_ctx.MakeOshParser(line_reader)
327
328 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
329 comp_lookup=comp_lookup,
330 arena=arena,
331 mem=mem,
332 aliases=aliases)
333
334 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
335 return cmd_ev
336
337
338def InitParseContext(arena=None,
339 ysh_grammar=None,
340 aliases=None,
341 parse_opts=None,
342 do_lossless=False):
343 arena = arena or MakeArena('<test_lib>')
344
345 if aliases is None:
346 aliases = {}
347
348 mem = state.Mem('', [], arena, [])
349 if parse_opts is None:
350 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
351
352 parse_ctx = parse_lib.ParseContext(arena,
353 parse_opts,
354 aliases,
355 ysh_grammar,
356 do_lossless=do_lossless)
357
358 return parse_ctx
359
360
361def InitWordParser(word_str, oil_at=False, arena=None):
362 arena = arena or MakeArena('<test_lib>')
363
364 mem = state.Mem('', [], arena, [])
365 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
366
367 # CUSTOM SETTING
368 mutable_opts.opt0_array[option_i.parse_at] = oil_at
369
370 loader = pyutil.GetResourceLoader()
371 ysh_grammar = pyutil.LoadYshGrammar(loader)
372 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
373 line_reader, _ = InitLexer(word_str, arena)
374 c_parser = parse_ctx.MakeOshParser(line_reader)
375 # Hack
376 return c_parser.w_parser
377
378
379def InitCommandParser(code_str, arena=None):
380 arena = arena or MakeArena('<test_lib>')
381
382 loader = pyutil.GetResourceLoader()
383 ysh_grammar = pyutil.LoadYshGrammar(loader)
384
385 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
386 line_reader, _ = InitLexer(code_str, arena)
387 c_parser = parse_ctx.MakeOshParser(line_reader)
388 return c_parser
389
390
391def SetLocalString(mem, name, s):
392 # type: (state.Mem, str, str) -> None
393 """Bind a local string."""
394 assert isinstance(s, str)
395 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)