OILS / core / shell.py View on Github | oilshell.org

1120 lines, 716 significant
1"""
2core/shell.py -- Entry point for the shell interpreter.
3"""
4from __future__ import print_function
5
6from errno import ENOENT
7import time as time_
8
9from _devbuild.gen import arg_types
10from _devbuild.gen.option_asdl import option_i, builtin_i
11from _devbuild.gen.runtime_asdl import scope_e
12from _devbuild.gen.syntax_asdl import (loc, source, source_t, IntParamBox,
13 debug_frame, debug_frame_t)
14from _devbuild.gen.value_asdl import (value, value_e)
15from core import alloc
16from core import comp_ui
17from core import dev
18from core import error
19from core import executor
20from core import completion
21from core import main_loop
22from core import pyos
23from core import process
24from core import pyutil
25from core import state
26from core import ui
27from core import util
28from core import vm
29
30from frontend import args
31from frontend import flag_def # side effect: flags are defined!
32
33unused1 = flag_def
34from frontend import flag_util
35from frontend import location
36from frontend import reader
37from frontend import parse_lib
38
39from builtin import assign_osh
40from builtin import bracket_osh
41from builtin import completion_osh
42from builtin import completion_ysh
43from builtin import dirs_osh
44from builtin import error_ysh
45from builtin import hay_ysh
46from builtin import io_osh
47from builtin import io_ysh
48from builtin import json_ysh
49from builtin import meta_osh
50from builtin import misc_osh
51from builtin import module_ysh
52from builtin import printf_osh
53from builtin import process_osh
54from builtin import pure_osh
55from builtin import pure_ysh
56from builtin import readline_osh
57from builtin import read_osh
58from builtin import trap_osh
59
60from builtin import func_eggex
61from builtin import func_hay
62from builtin import func_misc
63
64from builtin import method_dict
65from builtin import method_io
66from builtin import method_list
67from builtin import method_other
68from builtin import method_str
69
70from osh import cmd_eval
71from osh import glob_
72from osh import history
73from osh import prompt
74from osh import sh_expr_eval
75from osh import split
76from osh import word_eval
77
78from mycpp import mops
79from mycpp import mylib
80from mycpp.mylib import print_stderr, log
81from pylib import os_path
82from tools import deps
83from tools import fmt
84from tools import ysh_ify
85from ysh import expr_eval
86
87unused2 = log
88
89import libc
90import posix_ as posix
91
92from typing import List, Dict, Optional, TYPE_CHECKING, cast
93if TYPE_CHECKING:
94 from frontend.py_readline import Readline
95
96if mylib.PYTHON:
97 try:
98 from _devbuild.gen import help_meta # type: ignore
99 except ImportError:
100 help_meta = None
101
102
103def _InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup):
104 # type: (cmd_eval.CommandEvaluator, completion_osh.Complete, completion.Lookup) -> None
105
106 # register builtins and words
107 complete_builtin.Run(cmd_eval.MakeBuiltinArgv(['-E', '-A', 'command']))
108 # register path completion
109 # Add -o filenames? Or should that be automatic?
110 complete_builtin.Run(cmd_eval.MakeBuiltinArgv(['-D', '-A', 'file']))
111
112
113def _CompletionDemo(comp_lookup):
114 # type: (completion.Lookup) -> None
115
116 # Something for fun, to show off. Also: test that you don't repeatedly hit
117 # the file system / network / coprocess.
118 A1 = completion.TestAction(['foo.py', 'foo', 'bar.py'], 0.0)
119 l = [] # type: List[str]
120 for i in xrange(0, 5):
121 l.append('m%d' % i)
122
123 A2 = completion.TestAction(l, 0.1)
124 C1 = completion.UserSpec([A1, A2], [], [], completion.DefaultPredicate(),
125 '', '')
126 comp_lookup.RegisterName('slowc', {}, C1)
127
128
129def SourceStartupFile(
130 fd_state, # type: process.FdState
131 rc_path, # type: str
132 lang, # type: str
133 parse_ctx, # type: parse_lib.ParseContext
134 cmd_ev, # type: cmd_eval.CommandEvaluator
135 errfmt, # type: ui.ErrorFormatter
136):
137 # type: (...) -> None
138
139 # Right now this is called when the shell is interactive. (Maybe it should
140 # be called on login_shel too.)
141 #
142 # Terms:
143 # - interactive shell: Roughly speaking, no args or -c, and isatty() is true
144 # for stdin and stdout.
145 # - login shell: Started from the top level, e.g. from init or ssh.
146 #
147 # We're not going to copy everything bash does because it's too complex, but
148 # for reference:
149 # https://www.gnu.org/software/bash/manual/bash.html#Bash-Startup-Files
150 # Bash also has --login.
151
152 try:
153 f = fd_state.Open(rc_path)
154 except (IOError, OSError) as e:
155 # TODO: Could warn about nonexistent explicit --rcfile?
156 if e.errno != ENOENT:
157 raise # Goes to top level. Handle this better?
158 return
159
160 arena = parse_ctx.arena
161 rc_line_reader = reader.FileLineReader(f, arena)
162 rc_c_parser = parse_ctx.MakeOshParser(rc_line_reader)
163
164 with alloc.ctx_SourceCode(arena, source.SourcedFile(rc_path, loc.Missing)):
165 # TODO: handle status, e.g. 2 for ParseError
166 unused = main_loop.Batch(cmd_ev, rc_c_parser, errfmt)
167
168 f.close()
169
170
171class ShellOptHook(state.OptHook):
172
173 def __init__(self, readline):
174 # type: (Optional[Readline]) -> None
175 self.readline = readline
176
177 def OnChange(self, opt0_array, opt_name, b):
178 # type: (List[bool], str, bool) -> bool
179 """This method is called whenever an option is changed.
180
181 Returns success or failure.
182 """
183 if opt_name == 'vi' or opt_name == 'emacs':
184 # TODO: Replace with a hook? Just like setting LANG= can have a hook.
185 if self.readline:
186 self.readline.parse_and_bind("set editing-mode " + opt_name)
187 else:
188 print_stderr(
189 "Warning: Can't set option %r because shell wasn't compiled with GNU readline"
190 % opt_name)
191 return False
192
193 # Invert: they are mutually exclusive!
194 if opt_name == 'vi':
195 opt0_array[option_i.emacs] = not b
196 elif opt_name == 'emacs':
197 opt0_array[option_i.vi] = not b
198
199 return True
200
201
202def _SetGlobalFunc(mem, name, func):
203 # type: (state.Mem, str, vm._Callable) -> None
204 assert isinstance(func, vm._Callable), func
205
206 # Note: no location info for builtin functions?
207 mem.SetNamed(location.LName(name), value.BuiltinFunc(func),
208 scope_e.GlobalOnly)
209
210
211def InitAssignmentBuiltins(
212 mem, # type: state.Mem
213 procs, # type: Dict[str, value.Proc]
214 errfmt, # type: ui.ErrorFormatter
215):
216 # type: (...) -> Dict[int, vm._AssignBuiltin]
217
218 assign_b = {} # type: Dict[int, vm._AssignBuiltin]
219
220 new_var = assign_osh.NewVar(mem, procs, errfmt)
221 assign_b[builtin_i.declare] = new_var
222 assign_b[builtin_i.typeset] = new_var
223 assign_b[builtin_i.local] = new_var
224
225 assign_b[builtin_i.export_] = assign_osh.Export(mem, errfmt)
226 assign_b[builtin_i.readonly] = assign_osh.Readonly(mem, errfmt)
227
228 return assign_b
229
230
231class ShellFiles(object):
232
233 def __init__(self, lang, home_dir, mem, flag):
234 # type: (str, str, state.Mem, arg_types.main) -> None
235 assert lang in ('osh', 'ysh'), lang
236 self.lang = lang
237 self.home_dir = home_dir
238 self.mem = mem
239 self.flag = flag
240
241 def _HistVar(self):
242 # type: () -> str
243 return 'HISTFILE' if self.lang == 'osh' else 'YSH_HISTFILE'
244
245 def _DefaultHistoryFile(self):
246 # type: () -> str
247 return os_path.join(self.home_dir,
248 '.local/share/oils/%s_history' % self.lang)
249
250 def InitAfterLoadingEnv(self):
251 # type: () -> None
252
253 hist_var = self._HistVar()
254 if self.mem.GetValue(hist_var).tag() == value_e.Undef:
255 # Note: if the directory doesn't exist, GNU readline ignores
256 state.SetGlobalString(self.mem, hist_var,
257 self._DefaultHistoryFile())
258
259 def HistoryFile(self):
260 # type: () -> Optional[str]
261 # TODO: In non-strict mode we should try to cast the HISTFILE value to a
262 # string following bash's rules
263
264 UP_val = self.mem.GetValue(self._HistVar())
265 if UP_val.tag() == value_e.Str:
266 val = cast(value.Str, UP_val)
267 return val.s
268 else:
269 # Note: if HISTFILE is an array, bash will return ${HISTFILE[0]}
270 return None
271 #return self._DefaultHistoryFile()
272
273 # TODO: can we recover line information here?
274 # might be useful to show where HISTFILE was set
275 #raise error.Strict("$HISTFILE should only ever be a string", loc.Missing)
276
277
278def Main(
279 lang, # type: str
280 arg_r, # type: args.Reader
281 environ, # type: Dict[str, str]
282 login_shell, # type: bool
283 loader, # type: pyutil._ResourceLoader
284 readline, # type: Optional[Readline]
285):
286 # type: (...) -> int
287 """The full shell lifecycle. Used by bin/osh and bin/ysh.
288
289 Args:
290 lang: 'osh' or 'ysh'
291 login_shell: Was - on argv[0]?
292 loader: to get help, version, grammar, etc.
293 readline: optional GNU readline
294 """
295 # Differences between osh and ysh:
296 # - oshrc vs yshrc
297 # - shopt -s ysh:all
298 # - Prompt
299 # - --help
300
301 argv0 = arg_r.Peek()
302 assert argv0 is not None
303 arg_r.Next()
304
305 assert lang in ('osh', 'ysh'), lang
306
307 try:
308 attrs = flag_util.ParseMore('main', arg_r)
309 except error.Usage as e:
310 print_stderr('%s usage error: %s' % (lang, e.msg))
311 return 2
312 flag = arg_types.main(attrs.attrs)
313
314 arena = alloc.Arena()
315 errfmt = ui.ErrorFormatter()
316
317 if flag.help:
318 util.HelpFlag(loader, '%s-usage' % lang, mylib.Stdout())
319 return 0
320 if flag.version:
321 util.VersionFlag(loader, mylib.Stdout())
322 return 0
323
324 if flag.tool == 'cat-em':
325 paths = arg_r.Rest()
326
327 status = 0
328 for p in paths:
329 try:
330 contents = loader.Get(p)
331 print(contents)
332 except (OSError, IOError):
333 print_stderr("cat-em: %r not found" % p)
334 status = 1
335 return status
336
337 debug_stack = [] # type: List[debug_frame_t]
338 if arg_r.AtEnd():
339 dollar0 = argv0
340 else:
341 dollar0 = arg_r.Peek() # the script name, or the arg after -c
342
343 frame0 = debug_frame.Main(dollar0)
344 debug_stack.append(frame0)
345
346 script_name = arg_r.Peek() # type: Optional[str]
347 arg_r.Next()
348 mem = state.Mem(dollar0, arg_r.Rest(), arena, debug_stack)
349
350 opt_hook = ShellOptHook(readline)
351 # Note: only MutableOpts needs mem, so it's not a true circular dep.
352 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, opt_hook)
353 mem.exec_opts = exec_opts # circular dep
354 mutable_opts.Init()
355
356 version_str = pyutil.GetVersion(loader)
357 state.InitMem(mem, environ, version_str)
358
359 if attrs.show_options: # special case: sh -o
360 mutable_opts.ShowOptions([])
361 return 0
362
363 # Set these BEFORE processing flags, so they can be overridden.
364 if lang == 'ysh':
365 mutable_opts.SetAnyOption('ysh:all', True)
366
367 pure_osh.SetOptionsFromFlags(mutable_opts, attrs.opt_changes,
368 attrs.shopt_changes)
369
370 # feedback between runtime and parser
371 aliases = {} # type: Dict[str, str]
372
373 ysh_grammar = pyutil.LoadYshGrammar(loader)
374
375 if flag.do_lossless and not exec_opts.noexec():
376 raise error.Usage('--one-pass-parse requires noexec (-n)', loc.Missing)
377
378 # Tools always use one pass parse
379 # Note: osh --tool syntax-tree is like osh -n --one-pass-parse
380 do_lossless = True if len(flag.tool) else flag.do_lossless
381
382 parse_ctx = parse_lib.ParseContext(arena,
383 parse_opts,
384 aliases,
385 ysh_grammar,
386 do_lossless=do_lossless)
387
388 # Three ParseContext instances SHARE aliases.
389 comp_arena = alloc.Arena()
390 comp_arena.PushSource(source.Unused('completion'))
391 trail1 = parse_lib.Trail()
392 # do_lossless needs to be turned on to complete inside backticks. TODO:
393 # fix the issue where ` gets erased because it's not part of
394 # set_completer_delims().
395 comp_ctx = parse_lib.ParseContext(comp_arena,
396 parse_opts,
397 aliases,
398 ysh_grammar,
399 do_lossless=True)
400 comp_ctx.Init_Trail(trail1)
401
402 hist_arena = alloc.Arena()
403 hist_arena.PushSource(source.Unused('history'))
404 trail2 = parse_lib.Trail()
405 hist_ctx = parse_lib.ParseContext(hist_arena, parse_opts, aliases,
406 ysh_grammar)
407 hist_ctx.Init_Trail(trail2)
408
409 # Deps helps manages dependencies. These dependencies are circular:
410 # - cmd_ev and word_ev, arith_ev -- for command sub, arith sub
411 # - arith_ev and word_ev -- for $(( ${a} )) and $x$(( 1 ))
412 # - cmd_ev and builtins (which execute code, like eval)
413 # - prompt_ev needs word_ev for $PS1, which needs prompt_ev for @P
414 cmd_deps = cmd_eval.Deps()
415 cmd_deps.mutable_opts = mutable_opts
416
417 job_control = process.JobControl()
418 job_list = process.JobList()
419 fd_state = process.FdState(errfmt, job_control, job_list, mem, None, None)
420
421 my_pid = posix.getpid()
422
423 debug_path = ''
424 debug_dir = environ.get('OILS_DEBUG_DIR')
425 if flag.debug_file is not None:
426 # --debug-file takes precedence over OSH_DEBUG_DIR
427 debug_path = flag.debug_file
428 elif debug_dir is not None:
429 debug_path = os_path.join(debug_dir, '%d-osh.log' % my_pid)
430
431 if len(debug_path):
432 # This will be created as an empty file if it doesn't exist, or it could be
433 # a pipe.
434 try:
435 debug_f = util.DebugFile(
436 fd_state.OpenForWrite(debug_path)) # type: util._DebugFile
437 except (IOError, OSError) as e:
438 print_stderr("%s: Couldn't open %r: %s" %
439 (lang, debug_path, posix.strerror(e.errno)))
440 return 2
441 else:
442 debug_f = util.NullDebugFile()
443
444 if flag.xtrace_to_debug_file:
445 trace_f = debug_f
446 else:
447 trace_f = util.DebugFile(mylib.Stderr())
448
449 trace_dir = environ.get('OILS_TRACE_DIR', '')
450 dumps = environ.get('OILS_TRACE_DUMPS', '')
451 streams = environ.get('OILS_TRACE_STREAMS', '')
452 multi_trace = dev.MultiTracer(my_pid, trace_dir, dumps, streams, fd_state)
453
454 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, trace_f,
455 multi_trace)
456 fd_state.tracer = tracer # circular dep
457
458 signal_safe = pyos.InitSignalSafe()
459 trap_state = trap_osh.TrapState(signal_safe)
460
461 waiter = process.Waiter(job_list, exec_opts, signal_safe, tracer)
462 fd_state.waiter = waiter
463
464 cmd_deps.debug_f = debug_f
465
466 now = time_.time()
467 iso_stamp = time_.strftime("%Y-%m-%d %H:%M:%S", time_.localtime(now))
468
469 argv_buf = mylib.BufWriter()
470 dev.PrintShellArgv(arg_r.argv, argv_buf)
471
472 debug_f.writeln('%s [%d] Oils started with argv %s' %
473 (iso_stamp, my_pid, argv_buf.getvalue()))
474 if len(debug_path):
475 debug_f.writeln('Writing logs to %r' % debug_path)
476
477 interp = environ.get('OILS_HIJACK_SHEBANG', '')
478 search_path = state.SearchPath(mem)
479 ext_prog = process.ExternalProgram(interp, fd_state, errfmt, debug_f)
480
481 splitter = split.SplitContext(mem)
482 # TODO: This is instantiation is duplicated in osh/word_eval.py
483 globber = glob_.Globber(exec_opts)
484
485 # This could just be OILS_TRACE_DUMPS='crash:argv0'
486 crash_dump_dir = environ.get('OILS_CRASH_DUMP_DIR', '')
487 cmd_deps.dumper = dev.CrashDumper(crash_dump_dir, fd_state)
488
489 comp_lookup = completion.Lookup()
490
491 # Various Global State objects to work around readline interfaces
492 compopt_state = completion.OptionState()
493
494 comp_ui_state = comp_ui.State()
495 prompt_state = comp_ui.PromptState()
496
497 # The login program is supposed to set $HOME
498 # https://superuser.com/questions/271925/where-is-the-home-environment-variable-set
499 # state.InitMem(mem) must happen first
500 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
501 home_dir = tilde_ev.GetMyHomeDir()
502 if home_dir is None:
503 # TODO: print errno from getpwuid()
504 print_stderr("%s: Failed to get home dir from $HOME or getpwuid()" %
505 lang)
506 return 1
507
508 sh_files = ShellFiles(lang, home_dir, mem, flag)
509 sh_files.InitAfterLoadingEnv()
510
511 #
512 # Executor and Evaluators (are circularly dependent)
513 #
514
515 # Global proc namespace. Funcs are defined in the common variable
516 # namespace.
517 procs = {} # type: Dict[str, value.Proc]
518
519 builtins = {} # type: Dict[int, vm._Builtin]
520
521 # e.g. s->startswith()
522 methods = {} # type: Dict[int, Dict[str, vm._Callable]]
523
524 hay_state = hay_ysh.HayState()
525
526 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
527 hay_state, builtins, search_path,
528 ext_prog, waiter, tracer, job_control,
529 job_list, fd_state, trap_state, errfmt)
530
531 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
532 parse_ctx, errfmt)
533 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
534 parse_ctx, errfmt)
535 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
536 errfmt)
537 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
538 tilde_ev, splitter, errfmt)
539
540 assign_b = InitAssignmentBuiltins(mem, procs, errfmt)
541 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs, assign_b,
542 arena, cmd_deps, trap_state,
543 signal_safe)
544
545 # PromptEvaluator rendering is needed in non-interactive shells for @P.
546 prompt_ev = prompt.Evaluator(lang, version_str, parse_ctx, mem)
547 global_io = value.IO(cmd_ev, prompt_ev)
548 global_guts = value.Guts(None)
549
550 # Wire up circular dependencies.
551 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
552 prompt_ev, global_io, tracer)
553
554 unsafe_arith = sh_expr_eval.UnsafeArith(mem, exec_opts, mutable_opts,
555 parse_ctx, arith_ev, errfmt)
556 vm.InitUnsafeArith(mem, word_ev, unsafe_arith)
557
558 #
559 # Initialize Built-in Procs
560 #
561
562 b = builtins # short alias for initialization
563
564 if mylib.PYTHON:
565 if help_meta:
566 help_data = help_meta.TopicMetadata()
567 else:
568 help_data = {} # minimal build
569 else:
570 help_data = help_meta.TopicMetadata()
571 b[builtin_i.help] = misc_osh.Help(lang, loader, help_data, errfmt)
572
573 # Interpreter state
574 b[builtin_i.set] = pure_osh.Set(mutable_opts, mem)
575 b[builtin_i.shopt] = pure_osh.Shopt(mutable_opts, cmd_ev)
576
577 b[builtin_i.hash] = pure_osh.Hash(search_path) # not really pure
578 b[builtin_i.trap] = trap_osh.Trap(trap_state, parse_ctx, tracer, errfmt)
579
580 b[builtin_i.shvar] = pure_ysh.Shvar(mem, search_path, cmd_ev)
581 b[builtin_i.ctx] = pure_ysh.Ctx(mem, cmd_ev)
582 b[builtin_i.push_registers] = pure_ysh.PushRegisters(mem, cmd_ev)
583
584 # Hay
585 b[builtin_i.hay] = hay_ysh.Hay(hay_state, mutable_opts, mem, cmd_ev)
586 b[builtin_i.haynode] = hay_ysh.HayNode_(hay_state, mem, cmd_ev)
587
588 # Interpreter introspection
589 b[builtin_i.type] = meta_osh.Type(procs, aliases, search_path, errfmt)
590 b[builtin_i.builtin] = meta_osh.Builtin(shell_ex, errfmt)
591 b[builtin_i.command] = meta_osh.Command(shell_ex, procs, aliases,
592 search_path)
593 # Part of YSH, but similar to builtin/command
594 b[builtin_i.runproc] = meta_osh.RunProc(shell_ex, procs, errfmt)
595
596 # Meta builtins
597 source_builtin = meta_osh.Source(parse_ctx, search_path, cmd_ev, fd_state,
598 tracer, errfmt, loader)
599 b[builtin_i.source] = source_builtin
600 b[builtin_i.dot] = source_builtin
601 b[builtin_i.eval] = meta_osh.Eval(parse_ctx, exec_opts, cmd_ev, tracer,
602 errfmt)
603
604 # Module builtins
605 modules = {} # type: Dict[str, bool]
606 b[builtin_i.module] = module_ysh.Module(modules, exec_opts, errfmt)
607 b[builtin_i.is_main] = module_ysh.IsMain(mem)
608 b[builtin_i.use] = module_ysh.Use(mem, errfmt)
609
610 # Errors
611 b[builtin_i.error] = error_ysh.Error()
612 b[builtin_i.boolstatus] = error_ysh.BoolStatus(shell_ex, errfmt)
613 b[builtin_i.try_] = error_ysh.Try(mutable_opts, mem, cmd_ev, shell_ex,
614 errfmt)
615
616 # Pure builtins
617 true_ = pure_osh.Boolean(0)
618 b[builtin_i.colon] = true_ # a "special" builtin
619 b[builtin_i.true_] = true_
620 b[builtin_i.false_] = pure_osh.Boolean(1)
621
622 b[builtin_i.alias] = pure_osh.Alias(aliases, errfmt)
623 b[builtin_i.unalias] = pure_osh.UnAlias(aliases, errfmt)
624
625 b[builtin_i.getopts] = pure_osh.GetOpts(mem, errfmt)
626
627 b[builtin_i.shift] = assign_osh.Shift(mem)
628 b[builtin_i.unset] = assign_osh.Unset(mem, procs, unsafe_arith, errfmt)
629
630 b[builtin_i.append] = pure_ysh.Append(mem, errfmt)
631
632 # test / [ differ by need_right_bracket
633 b[builtin_i.test] = bracket_osh.Test(False, exec_opts, mem, errfmt)
634 b[builtin_i.bracket] = bracket_osh.Test(True, exec_opts, mem, errfmt)
635
636 # Output
637 b[builtin_i.echo] = io_osh.Echo(exec_opts)
638 b[builtin_i.printf] = printf_osh.Printf(mem, parse_ctx, unsafe_arith,
639 errfmt)
640 b[builtin_i.write] = io_ysh.Write(mem, errfmt)
641 b[builtin_i.fopen] = io_ysh.Fopen(mem, cmd_ev)
642
643 # (pp output format isn't stable)
644 b[builtin_i.pp] = io_ysh.Pp(mem, errfmt, procs, arena)
645
646 # Input
647 b[builtin_i.cat] = io_osh.Cat() # for $(<file)
648 b[builtin_i.read] = read_osh.Read(splitter, mem, parse_ctx, cmd_ev, errfmt)
649
650 mapfile = io_osh.MapFile(mem, errfmt, cmd_ev)
651 b[builtin_i.mapfile] = mapfile
652 b[builtin_i.readarray] = mapfile
653
654 # Dirs
655 dir_stack = dirs_osh.DirStack()
656 b[builtin_i.cd] = dirs_osh.Cd(mem, dir_stack, cmd_ev, errfmt)
657 b[builtin_i.pushd] = dirs_osh.Pushd(mem, dir_stack, errfmt)
658 b[builtin_i.popd] = dirs_osh.Popd(mem, dir_stack, errfmt)
659 b[builtin_i.dirs] = dirs_osh.Dirs(mem, dir_stack, errfmt)
660 b[builtin_i.pwd] = dirs_osh.Pwd(mem, errfmt)
661
662 b[builtin_i.times] = misc_osh.Times()
663
664 b[builtin_i.json] = json_ysh.Json(mem, errfmt, False)
665 b[builtin_i.json8] = json_ysh.Json(mem, errfmt, True)
666
667 ### Process builtins
668 b[builtin_i.exec_] = process_osh.Exec(mem, ext_prog, fd_state, search_path,
669 errfmt)
670 b[builtin_i.umask] = process_osh.Umask()
671 b[builtin_i.ulimit] = process_osh.Ulimit()
672 b[builtin_i.wait] = process_osh.Wait(waiter, job_list, mem, tracer, errfmt)
673
674 b[builtin_i.jobs] = process_osh.Jobs(job_list)
675 b[builtin_i.fg] = process_osh.Fg(job_control, job_list, waiter)
676 b[builtin_i.bg] = process_osh.Bg(job_list)
677
678 # Could be in process_ysh
679 b[builtin_i.fork] = process_osh.Fork(shell_ex)
680 b[builtin_i.forkwait] = process_osh.ForkWait(shell_ex)
681
682 # Interactive builtins depend on readline
683 b[builtin_i.bind] = readline_osh.Bind(readline, errfmt)
684 b[builtin_i.history] = readline_osh.History(readline, sh_files, errfmt,
685 mylib.Stdout())
686
687 # Completion
688 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
689 splitter, comp_lookup, help_data,
690 errfmt)
691 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
692 b[builtin_i.complete] = complete_builtin
693 b[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
694 b[builtin_i.compopt] = completion_osh.CompOpt(compopt_state, errfmt)
695 b[builtin_i.compadjust] = completion_osh.CompAdjust(mem)
696
697 comp_ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
698 tilde_ev, splitter, errfmt)
699
700 comp_ev.arith_ev = arith_ev
701 comp_ev.expr_ev = expr_ev
702 comp_ev.prompt_ev = prompt_ev
703 comp_ev.CheckCircularDeps()
704
705 root_comp = completion.RootCompleter(comp_ev, mem, comp_lookup,
706 compopt_state, comp_ui_state,
707 comp_ctx, debug_f)
708 b[builtin_i.compexport] = completion_ysh.CompExport(root_comp)
709
710 #
711 # Initialize Builtin-in Methods
712 #
713
714 methods[value_e.Str] = {
715 'startsWith': method_str.HasAffix(method_str.START),
716 'endsWith': method_str.HasAffix(method_str.END),
717 'trim': method_str.Trim(method_str.START | method_str.END),
718 'trimStart': method_str.Trim(method_str.START),
719 'trimEnd': method_str.Trim(method_str.END),
720
721 # These also have Unicode support
722 'upper': method_str.Upper(),
723 'lower': None,
724
725 # finds a substring, optional position to start at
726 'find': None,
727
728 # replace substring, OR an eggex
729 # takes count=3, the max number of replacements to do.
730 'replace': method_str.Replace(mem, expr_ev),
731
732 # Like Python's re.search, except we put it on the string object
733 # It's more consistent with Str->find(substring, pos=0)
734 # It returns value.Match() rather than an integer
735 'search': method_str.SearchMatch(method_str.SEARCH),
736
737 # like Python's re.match()
738 'leftMatch': method_str.SearchMatch(method_str.LEFT_MATCH),
739
740 # like Python's re.fullmatch(), not sure if we really need it
741 'fullMatch': None,
742 }
743 methods[value_e.Dict] = {
744 'get': None, # doesn't raise an error
745 'erase': None, # ensures it doesn't exist
746 'keys': method_dict.Keys(),
747 'values': method_dict.Values(),
748
749 # I think items() isn't as necessary because dicts are ordered?
750 # YSH code shouldn't use the List of Lists representation.
751
752 # could be d->tally() or d->increment(), but inc() is short
753 #
754 # call d->inc('mycounter')
755 # call d->inc('mycounter', 3)
756 'inc': None,
757
758 # call d->accum('mygroup', 'value')
759 'accum': None,
760 }
761 methods[value_e.List] = {
762 'reverse': method_list.Reverse(),
763 'append': method_list.Append(),
764 'extend': method_list.Extend(),
765 'pop': method_list.Pop(),
766 'insert': None, # insert object before index
767 'remove': None, # insert object before index
768 'indexOf': method_list.IndexOf(), # return first index of value, or -1
769 # Python list() has index(), which raises ValueError
770 # But this is consistent with Str->find(), and doesn't
771 # use exceptions
772 'join': func_misc.Join(), # both a method and a func
773 }
774
775 methods[value_e.Match] = {
776 'group': func_eggex.MatchMethod(func_eggex.G, expr_ev),
777 'start': func_eggex.MatchMethod(func_eggex.S, None),
778 'end': func_eggex.MatchMethod(func_eggex.E, None),
779 }
780
781 methods[value_e.IO] = {
782 # io->eval(myblock) is the functional version of eval (myblock)
783 # Should we also have expr->eval() instead of evalExpr?
784 'eval': None,
785
786 # identical to command sub
787 'captureStdout': None,
788 'promptVal': method_io.PromptVal(),
789 }
790
791 methods[value_e.Place] = {
792 # instead of setplace keyword
793 'setValue': method_other.SetValue(mem),
794 }
795
796 methods[value_e.Command] = {
797 # var x = ^(echo hi)
798 # Export source code and line number
799 # Useful for test frameworks and so forth
800 'export': None,
801 }
802
803 #
804 # Initialize Built-in Funcs
805 #
806
807 parse_hay = func_hay.ParseHay(fd_state, parse_ctx, errfmt)
808 eval_hay = func_hay.EvalHay(hay_state, mutable_opts, mem, cmd_ev)
809 hay_func = func_hay.HayFunc(hay_state)
810
811 _SetGlobalFunc(mem, 'parseHay', parse_hay)
812 _SetGlobalFunc(mem, 'evalHay', eval_hay)
813 _SetGlobalFunc(mem, '_hay', hay_func)
814
815 _SetGlobalFunc(mem, 'len', func_misc.Len())
816
817 g = func_eggex.MatchFunc(func_eggex.G, expr_ev, mem)
818 _SetGlobalFunc(mem, '_group', g)
819 _SetGlobalFunc(mem, '_match', g) # TODO: remove this backward compat alias
820 _SetGlobalFunc(mem, '_start', func_eggex.MatchFunc(func_eggex.S, None,
821 mem))
822 _SetGlobalFunc(mem, '_end', func_eggex.MatchFunc(func_eggex.E, None, mem))
823
824 _SetGlobalFunc(mem, 'join', func_misc.Join())
825 _SetGlobalFunc(mem, 'maybe', func_misc.Maybe())
826 _SetGlobalFunc(mem, 'type', func_misc.Type())
827 _SetGlobalFunc(mem, 'evalExpr', func_misc.EvalExpr(expr_ev))
828
829 # type conversions
830 _SetGlobalFunc(mem, 'bool', func_misc.Bool())
831 _SetGlobalFunc(mem, 'int', func_misc.Int())
832 _SetGlobalFunc(mem, 'float', func_misc.Float())
833 _SetGlobalFunc(mem, 'str', func_misc.Str_())
834 _SetGlobalFunc(mem, 'list', func_misc.List_())
835 _SetGlobalFunc(mem, 'dict', func_misc.Dict_())
836
837 # TODO: This should be Python style splitting
838 _SetGlobalFunc(mem, 'split', func_misc.Split(splitter))
839 _SetGlobalFunc(mem, 'shSplit', func_misc.Split(splitter))
840
841 _SetGlobalFunc(mem, 'glob', func_misc.Glob(globber))
842 _SetGlobalFunc(mem, 'shvarGet', func_misc.Shvar_get(mem))
843 _SetGlobalFunc(mem, 'getVar', func_misc.GetVar(mem))
844 _SetGlobalFunc(mem, 'assert_', func_misc.Assert())
845
846 _SetGlobalFunc(mem, 'toJson8', func_misc.ToJson8(True))
847 _SetGlobalFunc(mem, 'toJson', func_misc.ToJson8(False))
848
849 _SetGlobalFunc(mem, 'fromJson8', func_misc.FromJson8(True))
850 _SetGlobalFunc(mem, 'fromJson', func_misc.FromJson8(False))
851
852 mem.SetNamed(location.LName('_io'), global_io, scope_e.GlobalOnly)
853 mem.SetNamed(location.LName('_guts'), global_guts, scope_e.GlobalOnly)
854
855 #
856 # Is the shell interactive?
857 #
858
859 # History evaluation is a no-op if readline is None.
860 hist_ev = history.Evaluator(readline, hist_ctx, debug_f)
861
862 if flag.c is not None:
863 src = source.CFlag # type: source_t
864 line_reader = reader.StringLineReader(flag.c,
865 arena) # type: reader._Reader
866 if flag.i: # -c and -i can be combined
867 mutable_opts.set_interactive()
868
869 elif flag.i: # force interactive
870 src = source.Stdin(' -i')
871 line_reader = reader.InteractiveLineReader(arena, prompt_ev, hist_ev,
872 readline, prompt_state)
873 mutable_opts.set_interactive()
874
875 else:
876 if script_name is None:
877 if flag.headless:
878 src = source.Headless
879 line_reader = None # unused!
880 # Not setting '-i' flag for now. Some people's bashrc may want it?
881 else:
882 stdin_ = mylib.Stdin()
883 # --tool never starts a prompt
884 if len(flag.tool) == 0 and stdin_.isatty():
885 src = source.Interactive
886 line_reader = reader.InteractiveLineReader(
887 arena, prompt_ev, hist_ev, readline, prompt_state)
888 mutable_opts.set_interactive()
889 else:
890 src = source.Stdin('')
891 line_reader = reader.FileLineReader(stdin_, arena)
892 else:
893 src = source.MainFile(script_name)
894 try:
895 f = fd_state.Open(script_name)
896 except (IOError, OSError) as e:
897 print_stderr("%s: Couldn't open %r: %s" %
898 (lang, script_name, posix.strerror(e.errno)))
899 return 1
900 line_reader = reader.FileLineReader(f, arena)
901
902 # Pretend it came from somewhere else
903 if flag.location_str is not None:
904 src = source.Synthetic(flag.location_str)
905 assert line_reader is not None
906 location_start_line = mops.BigTruncate(flag.location_start_line)
907 if location_start_line != -1:
908 line_reader.SetLineOffset(location_start_line)
909
910 arena.PushSource(src)
911
912 # Calculate ~/.config/oils/oshrc or yshrc. Used for both -i and --headless
913 # We avoid cluttering the user's home directory. Some users may want to ln
914 # -s ~/.config/oils/oshrc ~/oshrc or ~/.oshrc.
915
916 # https://unix.stackexchange.com/questions/24347/why-do-some-applications-use-config-appname-for-their-config-data-while-other
917
918 config_dir = '.config/oils'
919 rc_paths = [] # type: List[str]
920 if not flag.norc and (flag.headless or exec_opts.interactive()):
921 # User's rcfile comes FIRST. Later we can add an 'after-rcdir' hook
922 rc_path = flag.rcfile
923 if rc_path is None:
924 rc_paths.append(
925 os_path.join(home_dir, '%s/%src' % (config_dir, lang)))
926 else:
927 rc_paths.append(rc_path)
928
929 # Load all files in ~/.config/oil/oshrc.d or oilrc.d
930 # This way "installers" can avoid mutating oshrc directly
931
932 rc_dir = flag.rcdir
933 if rc_dir is None:
934 rc_dir = os_path.join(home_dir, '%s/%src.d' % (config_dir, lang))
935
936 rc_paths.extend(libc.glob(os_path.join(rc_dir, '*')))
937 else:
938 if flag.rcfile is not None: # bash doesn't have this warning, but it's useful
939 print_stderr('%s warning: --rcfile ignored with --norc' % lang)
940 if flag.rcdir is not None:
941 print_stderr('%s warning: --rcdir ignored with --norc' % lang)
942
943 # Initialize even in non-interactive shell, for 'compexport'
944 _InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup)
945
946 if flag.headless:
947 state.InitInteractive(mem)
948 mutable_opts.set_redefine_proc_func()
949 mutable_opts.set_redefine_module()
950
951 # NOTE: rc files loaded AFTER _InitDefaultCompletions.
952 for rc_path in rc_paths:
953 with state.ctx_ThisDir(mem, rc_path):
954 try:
955 SourceStartupFile(fd_state, rc_path, lang, parse_ctx,
956 cmd_ev, errfmt)
957 except util.UserExit as e:
958 return e.status
959
960 loop = main_loop.Headless(cmd_ev, parse_ctx, errfmt)
961 try:
962 # TODO: What other exceptions happen here?
963 status = loop.Loop()
964 except util.UserExit as e:
965 status = e.status
966
967 # Same logic as interactive shell
968 mut_status = IntParamBox(status)
969 cmd_ev.MaybeRunExitTrap(mut_status)
970 status = mut_status.i
971
972 return status
973
974 # Note: headless mode above doesn't use c_parser
975 assert line_reader is not None
976 c_parser = parse_ctx.MakeOshParser(line_reader)
977
978 if exec_opts.interactive():
979 state.InitInteractive(mem)
980 # bash: 'set -o emacs' is the default only in the interactive shell
981 mutable_opts.set_emacs()
982 mutable_opts.set_redefine_proc_func()
983 mutable_opts.set_redefine_module()
984
985 if readline:
986 term_width = 0
987 if flag.completion_display == 'nice':
988 try:
989 term_width = libc.get_terminal_width()
990 except (IOError, OSError): # stdin not a terminal
991 pass
992
993 if term_width != 0:
994 display = comp_ui.NiceDisplay(
995 term_width, comp_ui_state, prompt_state, debug_f, readline,
996 signal_safe) # type: comp_ui._IDisplay
997 else:
998 display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
999 debug_f)
1000
1001 comp_ui.InitReadline(readline, sh_files.HistoryFile(), root_comp,
1002 display, debug_f)
1003
1004 _InitDefaultCompletions(cmd_ev, complete_builtin, comp_lookup)
1005 if flag.completion_demo:
1006 _CompletionDemo(comp_lookup)
1007
1008 else: # Without readline module
1009 display = comp_ui.MinimalDisplay(comp_ui_state, prompt_state,
1010 debug_f)
1011
1012 process.InitInteractiveShell() # Set signal handlers
1013
1014 # The interactive shell leads a process group which controls the terminal.
1015 # It MUST give up the terminal afterward, otherwise we get SIGTTIN /
1016 # SIGTTOU bugs.
1017 with process.ctx_TerminalControl(job_control, errfmt):
1018
1019 # NOTE: rc files loaded AFTER _InitDefaultCompletions.
1020 for rc_path in rc_paths:
1021 with state.ctx_ThisDir(mem, rc_path):
1022 try:
1023 SourceStartupFile(fd_state, rc_path, lang, parse_ctx,
1024 cmd_ev, errfmt)
1025 except util.UserExit as e:
1026 return e.status
1027
1028 assert line_reader is not None
1029 line_reader.Reset() # After sourcing startup file, render $PS1
1030
1031 prompt_plugin = prompt.UserPlugin(mem, parse_ctx, cmd_ev, errfmt)
1032 try:
1033 status = main_loop.Interactive(flag, cmd_ev, c_parser, display,
1034 prompt_plugin, waiter, errfmt)
1035 except util.UserExit as e:
1036 status = e.status
1037
1038 mut_status = IntParamBox(status)
1039 cmd_ev.MaybeRunExitTrap(mut_status)
1040 status = mut_status.i
1041
1042 if readline:
1043 hist_file = sh_files.HistoryFile()
1044 if hist_file is not None:
1045 try:
1046 readline.write_history_file(hist_file)
1047 except (IOError, OSError):
1048 pass
1049
1050 return status
1051
1052 if flag.rcfile is not None: # bash doesn't have this warning, but it's useful
1053 print_stderr('%s warning: --rcfile ignored in non-interactive shell' %
1054 lang)
1055 if flag.rcdir is not None:
1056 print_stderr('%s warning: --rcdir ignored in non-interactive shell' %
1057 lang)
1058
1059 #
1060 # Tools that use the OSH/YSH parsing mode, etc.
1061 #
1062
1063 # flag.tool is '' if nothing is passed
1064 # osh --tool syntax-tree is equivalent to osh -n --one-pass-parse
1065 tool_name = 'syntax-tree' if exec_opts.noexec() else flag.tool
1066
1067 if len(tool_name):
1068 # Don't save tokens becaues it's slow
1069 if tool_name != 'syntax-tree':
1070 arena.SaveTokens()
1071
1072 try:
1073 node = main_loop.ParseWholeFile(c_parser)
1074 except error.Parse as e:
1075 errfmt.PrettyPrintError(e)
1076 return 2
1077
1078 if tool_name == 'syntax-tree':
1079 ui.PrintAst(node, flag)
1080
1081 elif tool_name == 'tokens':
1082 ysh_ify.PrintTokens(arena)
1083
1084 elif tool_name == 'lossless-cat': # for test/lossless.sh
1085 ysh_ify.LosslessCat(arena)
1086
1087 elif tool_name == 'fmt':
1088 fmt.Format(arena, node)
1089
1090 elif tool_name == 'ysh-ify':
1091 ysh_ify.Ysh_ify(arena, node)
1092
1093 elif tool_name == 'deps':
1094 if mylib.PYTHON:
1095 deps.Deps(node)
1096
1097 else:
1098 raise AssertionError(tool_name) # flag parser validated it
1099
1100 return 0
1101
1102 #
1103 # Run a shell script
1104 #
1105
1106 with state.ctx_ThisDir(mem, script_name):
1107 try:
1108 status = main_loop.Batch(cmd_ev,
1109 c_parser,
1110 errfmt,
1111 cmd_flags=cmd_eval.IsMainProgram)
1112 except util.UserExit as e:
1113 status = e.status
1114 mut_status = IntParamBox(status)
1115 cmd_ev.MaybeRunExitTrap(mut_status)
1116
1117 multi_trace.WriteDumps()
1118
1119 # NOTE: We haven't closed the file opened with fd_state.Open
1120 return mut_status.i