OILS / core / test_lib.py View on Github | oilshell.org

399 lines, 279 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value
19from asdl import pybase
20from builtin import assign_osh
21from builtin import completion_osh
22from builtin import hay_ysh
23from builtin import io_osh
24from builtin import pure_osh
25from builtin import readline_osh
26from builtin import trap_osh
27from core import alloc
28from core import completion
29from core import dev
30from core import executor
31from core import main_loop
32from core import optview
33from core import process
34from core import pyos
35from core import pyutil
36from core import state
37from display import ui
38from core import util
39from core import vm
40from frontend import lexer
41from frontend import location
42from frontend import parse_lib
43from frontend import reader
44from osh import cmd_eval
45from osh import prompt
46from osh import sh_expr_eval
47from osh import split
48from osh import word_eval
49from ysh import expr_eval
50from mycpp import mylib
51
52import posix_ as posix
53
54
55def MakeBuiltinArgv(argv):
56 return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None)
57
58
59def FakeTok(id_, val):
60 # type: (int, str) -> Token
61 src = source.Interactive
62 line = SourceLine(1, val, src)
63 return Token(id_, len(val), 0, line, None)
64
65
66def PrintableString(s):
67 """For pretty-printing in tests."""
68 if all(c in string.printable for c in s):
69 return s
70 return repr(s)
71
72
73def TokensEqual(left, right):
74 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
75
76 if left.id != right.id:
77 return False
78
79 if left.line is not None:
80 left_str = lexer.TokenVal(left)
81 else:
82 left_str = None
83
84 if right.line is not None:
85 right_str = lexer.TokenVal(right)
86 else:
87 right_str = None
88
89 # Better error message sometimes:
90 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
91 return left_str == right_str
92
93
94def TokenWordsEqual(left, right):
95 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
96 return TokensEqual(left.token, right.token)
97 #return left == right
98
99
100def AsdlEqual(left, right):
101 """Check if generated ASDL instances are equal.
102
103 We don't use equality in the actual code, so this is relegated to
104 test_lib.
105 """
106 if left is None and right is None:
107 return True
108
109 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
110 return left == right
111
112 if isinstance(left, list):
113 if len(left) != len(right):
114 return False
115 for a, b in zip(left, right):
116 if not AsdlEqual(a, b):
117 return False
118 return True
119
120 if isinstance(left, pybase.CompoundObj):
121 if left.tag() != right.tag():
122 return False
123
124 field_names = left.__slots__ # hack for now
125 for name in field_names:
126 # Special case: we are not testing locations right now.
127 if name == 'span_id':
128 continue
129 a = getattr(left, name)
130 b = getattr(right, name)
131 if not AsdlEqual(a, b):
132 return False
133
134 return True
135
136 raise AssertionError(left)
137
138
139def AssertAsdlEqual(test, left, right):
140 test.assertTrue(AsdlEqual(left, right),
141 'Expected %s, got %s' % (left, right))
142
143
144def MakeArena(source_name):
145 arena = alloc.Arena(save_tokens=True)
146 arena.PushSource(source.MainFile(source_name))
147 return arena
148
149
150def InitLineLexer(s, arena):
151 line_lexer = lexer.LineLexer(arena)
152 src = source.Interactive
153 line_lexer.Reset(SourceLine(1, s, src), 0)
154 return line_lexer
155
156
157def InitLexer(s, arena):
158 """For tests only."""
159 line_lexer = lexer.LineLexer(arena)
160 line_reader = reader.StringLineReader(s, arena)
161 lx = lexer.Lexer(line_lexer, line_reader)
162 return line_reader, lx
163
164
165def InitWordEvaluator(exec_opts=None):
166 arena = MakeArena('<InitWordEvaluator>')
167 mem = state.Mem('', [], arena, [])
168
169 if exec_opts is None:
170 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
171 mem.exec_opts = exec_opts # circular dep
172 state.InitMem(mem, {}, '0.1')
173 mutable_opts.Init()
174 else:
175 mutable_opts = None
176
177 cmd_deps = cmd_eval.Deps()
178 cmd_deps.trap_nodes = []
179
180 splitter = split.SplitContext(mem)
181 errfmt = ui.ErrorFormatter()
182
183 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
184 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
185 tilde_ev, splitter, errfmt)
186 return ev
187
188
189def InitCommandEvaluator(parse_ctx=None,
190 comp_lookup=None,
191 arena=None,
192 mem=None,
193 aliases=None,
194 ext_prog=None):
195
196 opt0_array = state.InitOpts()
197 opt_stacks = [None] * option_i.ARRAY_SIZE
198 if parse_ctx:
199 arena = parse_ctx.arena
200 else:
201 parse_ctx = InitParseContext()
202
203 mem = mem or state.Mem('', [], arena, [])
204 exec_opts = optview.Exec(opt0_array, opt_stacks)
205 mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
206 mem.exec_opts = exec_opts
207 state.InitMem(mem, {}, '0.1')
208 mutable_opts.Init()
209
210 # No 'readline' in the tests.
211
212 errfmt = ui.ErrorFormatter()
213 job_control = process.JobControl()
214 job_list = process.JobList()
215 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None,
216 exec_opts)
217 aliases = {} if aliases is None else aliases
218 procs = state.Procs(mem)
219 methods = {}
220
221 compopt_state = completion.OptionState()
222 comp_lookup = comp_lookup or completion.Lookup()
223
224 readline = None # simulate not having it
225
226 new_var = assign_osh.NewVar(mem, procs, exec_opts, errfmt)
227 assign_builtins = {
228 builtin_i.declare: new_var,
229 builtin_i.typeset: new_var,
230 builtin_i.local: new_var,
231 builtin_i.export_: assign_osh.Export(mem, errfmt),
232 builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
233 }
234 builtins = { # Lookup
235 builtin_i.echo: io_osh.Echo(exec_opts),
236 builtin_i.shift: assign_osh.Shift(mem),
237
238 builtin_i.history: readline_osh.History(
239 readline,
240 mem,
241 errfmt,
242 mylib.Stdout(),
243 ),
244
245 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
246 builtin_i.compadjust: completion_osh.CompAdjust(mem),
247
248 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
249 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
250 }
251
252 debug_f = util.DebugFile(sys.stderr)
253 cmd_deps = cmd_eval.Deps()
254 cmd_deps.mutable_opts = mutable_opts
255
256 search_path = state.SearchPath(mem)
257
258 ext_prog = \
259 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
260
261 cmd_deps.dumper = dev.CrashDumper('', fd_state)
262 cmd_deps.debug_f = debug_f
263
264 splitter = split.SplitContext(mem)
265
266 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
267 parse_ctx, errfmt)
268 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
269 parse_ctx, errfmt)
270 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
271 errfmt)
272 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
273 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
274 tilde_ev, splitter, errfmt)
275 signal_safe = pyos.InitSignalSafe()
276 trap_state = trap_osh.TrapState(signal_safe)
277 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
278 assign_builtins, arena, cmd_deps,
279 trap_state, signal_safe)
280
281 multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
282 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
283 multi_trace)
284 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
285
286 hay_state = hay_ysh.HayState()
287 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
288 hay_state, builtins, search_path,
289 ext_prog, waiter, tracer, job_control,
290 job_list, fd_state, trap_state, errfmt)
291
292 assert cmd_ev.mutable_opts is not None, cmd_ev
293 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
294
295 global_io = value.IO(cmd_ev, prompt_ev)
296 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
297 prompt_ev, global_io, tracer)
298
299 try:
300 from _devbuild.gen.help_meta import TOPICS
301 except ImportError:
302 TOPICS = None # minimal dev build
303 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
304 splitter, comp_lookup, TOPICS,
305 errfmt)
306
307 # Add some builtins that depend on the executor!
308 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
309 builtins[builtin_i.complete] = complete_builtin
310 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
311
312 return cmd_ev
313
314
315def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
316 """Unit tests can evaluate code strings and then use the resulting
317 CommandEvaluator."""
318 arena = parse_ctx.arena
319 errfmt = ui.ErrorFormatter()
320
321 comp_lookup = comp_lookup or completion.Lookup()
322 mem = mem or state.Mem('', [], arena, [])
323 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
324 mem.exec_opts = exec_opts
325
326 state.InitMem(mem, {}, '0.1')
327 mutable_opts.Init()
328
329 line_reader, _ = InitLexer(code_str, arena)
330 c_parser = parse_ctx.MakeOshParser(line_reader)
331
332 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
333 comp_lookup=comp_lookup,
334 arena=arena,
335 mem=mem,
336 aliases=aliases)
337
338 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
339 return cmd_ev
340
341
342def InitParseContext(arena=None,
343 ysh_grammar=None,
344 aliases=None,
345 parse_opts=None,
346 do_lossless=False):
347 arena = arena or MakeArena('<test_lib>')
348
349 if aliases is None:
350 aliases = {}
351
352 mem = state.Mem('', [], arena, [])
353 if parse_opts is None:
354 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
355
356 parse_ctx = parse_lib.ParseContext(arena,
357 parse_opts,
358 aliases,
359 ysh_grammar,
360 do_lossless=do_lossless)
361
362 return parse_ctx
363
364
365def InitWordParser(word_str, oil_at=False, arena=None):
366 arena = arena or MakeArena('<test_lib>')
367
368 mem = state.Mem('', [], arena, [])
369 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
370
371 # CUSTOM SETTING
372 mutable_opts.opt0_array[option_i.parse_at] = oil_at
373
374 loader = pyutil.GetResourceLoader()
375 ysh_grammar = pyutil.LoadYshGrammar(loader)
376 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
377 line_reader, _ = InitLexer(word_str, arena)
378 c_parser = parse_ctx.MakeOshParser(line_reader)
379 # Hack
380 return c_parser.w_parser
381
382
383def InitCommandParser(code_str, arena=None):
384 arena = arena or MakeArena('<test_lib>')
385
386 loader = pyutil.GetResourceLoader()
387 ysh_grammar = pyutil.LoadYshGrammar(loader)
388
389 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
390 line_reader, _ = InitLexer(code_str, arena)
391 c_parser = parse_ctx.MakeOshParser(line_reader)
392 return c_parser
393
394
395def SetLocalString(mem, name, s):
396 # type: (state.Mem, str, str) -> None
397 """Bind a local string."""
398 assert isinstance(s, str)
399 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)