OILS / core / process.py View on Github | oilshell.org

1951 lines, 942 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from display import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_NONBLOCK,
58 O_NOCTTY,
59 O_RDONLY,
60 O_RDWR,
61 O_WRONLY,
62 O_TRUNC,
63)
64
65from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
66
67if TYPE_CHECKING:
68 from _devbuild.gen.runtime_asdl import cmd_value
69 from _devbuild.gen.syntax_asdl import command_t
70 from builtin import trap_osh
71 from core import optview
72 from core.util import _DebugFile
73 from osh.cmd_eval import CommandEvaluator
74
75NO_FD = -1
76
77# Minimum file descriptor that the shell can use. Other descriptors can be
78# directly used by user programs, e.g. exec 9>&1
79#
80# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
81# This is a compromise between bash (unlimited, but requires crazy
82# bookkeeping), and dash/zsh (10) and mksh (24)
83_SHELL_MIN_FD = 100
84
85# Style for 'jobs' builtin
86STYLE_DEFAULT = 0
87STYLE_LONG = 1
88STYLE_PID_ONLY = 2
89
90# To save on allocations in JobList::GetJobWithSpec()
91CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
92
93
94class ctx_FileCloser(object):
95
96 def __init__(self, f):
97 # type: (mylib.LineReader) -> None
98 self.f = f
99
100 def __enter__(self):
101 # type: () -> None
102 pass
103
104 def __exit__(self, type, value, traceback):
105 # type: (Any, Any, Any) -> None
106 self.f.close()
107
108
109def InitInteractiveShell():
110 # type: () -> None
111 """Called when initializing an interactive shell."""
112
113 # The shell itself should ignore Ctrl-\.
114 pyos.Sigaction(SIGQUIT, SIG_IGN)
115
116 # This prevents Ctrl-Z from suspending OSH in interactive mode.
117 pyos.Sigaction(SIGTSTP, SIG_IGN)
118
119 # More signals from
120 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
121 # (but not SIGCHLD)
122 pyos.Sigaction(SIGTTOU, SIG_IGN)
123 pyos.Sigaction(SIGTTIN, SIG_IGN)
124
125 # Register a callback to receive terminal width changes.
126 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
127
128 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
129 # read() have to handle it
130 pyos.RegisterSignalInterest(SIGWINCH)
131
132
133def SaveFd(fd):
134 # type: (int) -> int
135 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
136 return saved
137
138
139class _RedirFrame(object):
140
141 def __init__(self, saved_fd, orig_fd, forget):
142 # type: (int, int, bool) -> None
143 self.saved_fd = saved_fd
144 self.orig_fd = orig_fd
145 self.forget = forget
146
147
148class _FdFrame(object):
149
150 def __init__(self):
151 # type: () -> None
152 self.saved = [] # type: List[_RedirFrame]
153 self.need_wait = [] # type: List[Process]
154
155 def Forget(self):
156 # type: () -> None
157 """For exec 1>&2."""
158 for rf in reversed(self.saved):
159 if rf.saved_fd != NO_FD and rf.forget:
160 posix.close(rf.saved_fd)
161
162 del self.saved[:] # like list.clear() in Python 3.3
163 del self.need_wait[:]
164
165 def __repr__(self):
166 # type: () -> str
167 return '<_FdFrame %s>' % self.saved
168
169
170class FdState(object):
171 """File descriptor state for the current process.
172
173 For example, you can do 'myfunc > out.txt' without forking. Child
174 processes inherit our state.
175 """
176
177 def __init__(
178 self,
179 errfmt, # type: ui.ErrorFormatter
180 job_control, # type: JobControl
181 job_list, # type: JobList
182 mem, #type: state.Mem
183 tracer, # type: Optional[dev.Tracer]
184 waiter, # type: Optional[Waiter]
185 ):
186 # type: (...) -> None
187 """
188 Args:
189 errfmt: for errors
190 job_list: For keeping track of _HereDocWriterThunk
191 """
192 self.errfmt = errfmt
193 self.job_control = job_control
194 self.job_list = job_list
195 self.cur_frame = _FdFrame() # for the top level
196 self.stack = [self.cur_frame]
197 self.mem = mem
198 self.tracer = tracer
199 self.waiter = waiter
200
201 def Open(self, path):
202 # type: (str) -> mylib.LineReader
203 """Opens a path for read, but moves it out of the reserved 3-9 fd
204 range.
205
206 Returns:
207 A Python file object. The caller is responsible for Close().
208
209 Raises:
210 IOError or OSError if the path can't be found. (This is Python-induced wart)
211 """
212 fd_mode = O_RDONLY
213 f = self._Open(path, 'r', fd_mode)
214
215 # Hacky downcast
216 return cast('mylib.LineReader', f)
217
218 # used for util.DebugFile
219 def OpenForWrite(self, path):
220 # type: (str) -> mylib.Writer
221 fd_mode = O_CREAT | O_RDWR
222 f = self._Open(path, 'w', fd_mode)
223
224 # Hacky downcast
225 return cast('mylib.Writer', f)
226
227 def _Open(self, path, c_mode, fd_mode):
228 # type: (str, str, int) -> IO[str]
229 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
230
231 # Immediately move it to a new location
232 new_fd = SaveFd(fd)
233 posix.close(fd)
234
235 # Return a Python file handle
236 f = posix.fdopen(new_fd, c_mode) # may raise IOError
237 return f
238
239 def _WriteFdToMem(self, fd_name, fd):
240 # type: (str, int) -> None
241 if self.mem:
242 # setvar, not setref
243 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
244 value.Str(str(fd)))
245
246 def _ReadFdFromMem(self, fd_name):
247 # type: (str) -> int
248 val = self.mem.GetValue(fd_name)
249 if val.tag() == value_e.Str:
250 try:
251 return int(cast(value.Str, val).s)
252 except ValueError:
253 return NO_FD
254 return NO_FD
255
256 def _PushSave(self, fd):
257 # type: (int) -> bool
258 """Save fd to a new location and remember to restore it later."""
259 #log('---- _PushSave %s', fd)
260 ok = True
261 try:
262 new_fd = SaveFd(fd)
263 except (IOError, OSError) as e:
264 ok = False
265 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
266 # open.
267 # This seems to be ignored in dash too in savefd()?
268 if e.errno != EBADF:
269 raise
270 if ok:
271 posix.close(fd)
272 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
273 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
274 else:
275 # if we got EBADF, we still need to close the original on Pop()
276 self._PushClose(fd)
277
278 return ok
279
280 def _PushDup(self, fd1, blame_loc):
281 # type: (int, redir_loc_t) -> int
282 """Save fd2 in a higher range, and dup fd1 onto fd2.
283
284 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
285 """
286 UP_loc = blame_loc
287 if blame_loc.tag() == redir_loc_e.VarName:
288 fd2_name = cast(redir_loc.VarName, UP_loc).name
289 try:
290 # F_DUPFD: GREATER than range
291 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
292 except (IOError, OSError) as e:
293 if e.errno == EBADF:
294 print_stderr('F_DUPFD fd %d: %s' %
295 (fd1, pyutil.strerror(e)))
296 return NO_FD
297 else:
298 raise # this redirect failed
299
300 self._WriteFdToMem(fd2_name, new_fd)
301
302 elif blame_loc.tag() == redir_loc_e.Fd:
303 fd2 = cast(redir_loc.Fd, UP_loc).fd
304
305 if fd1 == fd2:
306 # The user could have asked for it to be open on descriptor 3, but open()
307 # already returned 3, e.g. echo 3>out.txt
308 return NO_FD
309
310 # Check the validity of fd1 before _PushSave(fd2)
311 try:
312 fcntl_.fcntl(fd1, F_GETFD)
313 except (IOError, OSError) as e:
314 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
315 raise
316
317 need_restore = self._PushSave(fd2)
318
319 #log('==== dup2 %s %s\n' % (fd1, fd2))
320 try:
321 posix.dup2(fd1, fd2)
322 except (IOError, OSError) as e:
323 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
324 print_stderr('dup2(%d, %d): %s' %
325 (fd1, fd2, pyutil.strerror(e)))
326
327 # Restore and return error
328 if need_restore:
329 rf = self.cur_frame.saved.pop()
330 posix.dup2(rf.saved_fd, rf.orig_fd)
331 posix.close(rf.saved_fd)
332
333 raise # this redirect failed
334
335 new_fd = fd2
336
337 else:
338 raise AssertionError()
339
340 return new_fd
341
342 def _PushCloseFd(self, blame_loc):
343 # type: (redir_loc_t) -> bool
344 """For 2>&-"""
345 # exec {fd}>&- means close the named descriptor
346
347 UP_loc = blame_loc
348 if blame_loc.tag() == redir_loc_e.VarName:
349 fd_name = cast(redir_loc.VarName, UP_loc).name
350 fd = self._ReadFdFromMem(fd_name)
351 if fd == NO_FD:
352 return False
353
354 elif blame_loc.tag() == redir_loc_e.Fd:
355 fd = cast(redir_loc.Fd, UP_loc).fd
356
357 else:
358 raise AssertionError()
359
360 self._PushSave(fd)
361
362 return True
363
364 def _PushClose(self, fd):
365 # type: (int) -> None
366 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
367
368 def _PushWait(self, proc):
369 # type: (Process) -> None
370 self.cur_frame.need_wait.append(proc)
371
372 def _ApplyRedirect(self, r):
373 # type: (RedirValue) -> None
374 arg = r.arg
375 UP_arg = arg
376 with tagswitch(arg) as case:
377
378 if case(redirect_arg_e.Path):
379 arg = cast(redirect_arg.Path, UP_arg)
380
381 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
382 # NOTE: This is different than >| because it respects noclobber, but
383 # that option is almost never used. See test/wild.sh.
384 mode = O_CREAT | O_WRONLY | O_TRUNC
385 elif r.op_id == Id.Redir_Clobber: # >|
386 mode = O_CREAT | O_WRONLY | O_TRUNC
387 elif r.op_id in (Id.Redir_DGreat,
388 Id.Redir_AndDGreat): # >> &>>
389 mode = O_CREAT | O_WRONLY | O_APPEND
390 elif r.op_id == Id.Redir_Less: # <
391 mode = O_RDONLY
392 elif r.op_id == Id.Redir_LessGreat: # <>
393 mode = O_CREAT | O_RDWR
394 else:
395 raise NotImplementedError(r.op_id)
396
397 # NOTE: 0666 is affected by umask, all shells use it.
398 try:
399 open_fd = posix.open(arg.filename, mode, 0o666)
400 except (IOError, OSError) as e:
401 self.errfmt.Print_("Can't open %r: %s" %
402 (arg.filename, pyutil.strerror(e)),
403 blame_loc=r.op_loc)
404 raise # redirect failed
405
406 new_fd = self._PushDup(open_fd, r.loc)
407 if new_fd != NO_FD:
408 posix.close(open_fd)
409
410 # Now handle &> and &>> and their variants. These pairs are the same:
411 #
412 # stdout_stderr.py &> out-err.txt
413 # stdout_stderr.py > out-err.txt 2>&1
414 #
415 # stdout_stderr.py 3&> out-err.txt
416 # stdout_stderr.py 3> out-err.txt 2>&3
417 #
418 # Ditto for {fd}> and {fd}&>
419
420 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
421 self._PushDup(new_fd, redir_loc.Fd(2))
422
423 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
424 arg = cast(redirect_arg.CopyFd, UP_arg)
425
426 if r.op_id == Id.Redir_GreatAnd: # 1>&2
427 self._PushDup(arg.target_fd, r.loc)
428
429 elif r.op_id == Id.Redir_LessAnd: # 0<&5
430 # The only difference between >& and <& is the default file
431 # descriptor argument.
432 self._PushDup(arg.target_fd, r.loc)
433
434 else:
435 raise NotImplementedError()
436
437 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
438 arg = cast(redirect_arg.MoveFd, UP_arg)
439 new_fd = self._PushDup(arg.target_fd, r.loc)
440 if new_fd != NO_FD:
441 posix.close(arg.target_fd)
442
443 UP_loc = r.loc
444 if r.loc.tag() == redir_loc_e.Fd:
445 fd = cast(redir_loc.Fd, UP_loc).fd
446 else:
447 fd = NO_FD
448
449 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
450
451 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
452 self._PushCloseFd(r.loc)
453
454 elif case(redirect_arg_e.HereDoc):
455 arg = cast(redirect_arg.HereDoc, UP_arg)
456
457 # NOTE: Do these descriptors have to be moved out of the range 0-9?
458 read_fd, write_fd = posix.pipe()
459
460 self._PushDup(read_fd, r.loc) # stdin is now the pipe
461
462 # We can't close like we do in the filename case above? The writer can
463 # get a "broken pipe".
464 self._PushClose(read_fd)
465
466 thunk = _HereDocWriterThunk(write_fd, arg.body)
467
468 # Use PIPE_SIZE to save a process in the case of small here
469 # docs, which are the common case. (dash does this.)
470
471 # Note: could instrument this to see how often it happens.
472 # Though strace -ff can also work.
473 start_process = len(arg.body) > 4096
474 #start_process = True
475
476 if start_process:
477 here_proc = Process(thunk, self.job_control, self.job_list,
478 self.tracer)
479
480 # NOTE: we could close the read pipe here, but it doesn't really
481 # matter because we control the code.
482 here_proc.StartProcess(trace.HereDoc)
483 #log('Started %s as %d', here_proc, pid)
484 self._PushWait(here_proc)
485
486 # Now that we've started the child, close it in the parent.
487 posix.close(write_fd)
488
489 else:
490 posix.write(write_fd, arg.body)
491 posix.close(write_fd)
492
493 def Push(self, redirects, err_out):
494 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
495 """Apply a group of redirects and remember to undo them."""
496
497 #log('> fd_state.Push %s', redirects)
498 new_frame = _FdFrame()
499 self.stack.append(new_frame)
500 self.cur_frame = new_frame
501
502 for r in redirects:
503 #log('apply %s', r)
504 with ui.ctx_Location(self.errfmt, r.op_loc):
505 try:
506 self._ApplyRedirect(r)
507 except (IOError, OSError) as e:
508 err_out.append(e)
509 # This can fail too
510 self.Pop(err_out)
511 return # for bad descriptor, etc.
512
513 def PushStdinFromPipe(self, r):
514 # type: (int) -> bool
515 """Save the current stdin and make it come from descriptor 'r'.
516
517 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
518 semantics of
519
520 echo foo | read line; echo $line
521 """
522 new_frame = _FdFrame()
523 self.stack.append(new_frame)
524 self.cur_frame = new_frame
525
526 self._PushDup(r, redir_loc.Fd(0))
527 return True
528
529 def Pop(self, err_out):
530 # type: (List[error.IOError_OSError]) -> None
531 frame = self.stack.pop()
532 #log('< Pop %s', frame)
533 for rf in reversed(frame.saved):
534 if rf.saved_fd == NO_FD:
535 #log('Close %d', orig)
536 try:
537 posix.close(rf.orig_fd)
538 except (IOError, OSError) as e:
539 err_out.append(e)
540 log('Error closing descriptor %d: %s', rf.orig_fd,
541 pyutil.strerror(e))
542 return
543 else:
544 try:
545 posix.dup2(rf.saved_fd, rf.orig_fd)
546 except (IOError, OSError) as e:
547 err_out.append(e)
548 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
549 pyutil.strerror(e))
550 #log('fd state:')
551 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
552 return
553 posix.close(rf.saved_fd)
554 #log('dup2 %s %s', saved, orig)
555
556 # Wait for here doc processes to finish.
557 for proc in frame.need_wait:
558 unused_status = proc.Wait(self.waiter)
559
560 def MakePermanent(self):
561 # type: () -> None
562 self.cur_frame.Forget()
563
564
565class ChildStateChange(object):
566
567 def __init__(self):
568 # type: () -> None
569 """Empty constructor for mycpp."""
570 pass
571
572 def Apply(self):
573 # type: () -> None
574 raise NotImplementedError()
575
576 def ApplyFromParent(self, proc):
577 # type: (Process) -> None
578 """Noop for all state changes other than SetPgid for mycpp."""
579 pass
580
581
582class StdinFromPipe(ChildStateChange):
583
584 def __init__(self, pipe_read_fd, w):
585 # type: (int, int) -> None
586 self.r = pipe_read_fd
587 self.w = w
588
589 def __repr__(self):
590 # type: () -> str
591 return '<StdinFromPipe %d %d>' % (self.r, self.w)
592
593 def Apply(self):
594 # type: () -> None
595 posix.dup2(self.r, 0)
596 posix.close(self.r) # close after dup
597
598 posix.close(self.w) # we're reading from the pipe, not writing
599 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
600
601
602class StdoutToPipe(ChildStateChange):
603
604 def __init__(self, r, pipe_write_fd):
605 # type: (int, int) -> None
606 self.r = r
607 self.w = pipe_write_fd
608
609 def __repr__(self):
610 # type: () -> str
611 return '<StdoutToPipe %d %d>' % (self.r, self.w)
612
613 def Apply(self):
614 # type: () -> None
615 posix.dup2(self.w, 1)
616 posix.close(self.w) # close after dup
617
618 posix.close(self.r) # we're writing to the pipe, not reading
619 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
620
621
622INVALID_PGID = -1
623# argument to setpgid() that means the process is its own leader
624OWN_LEADER = 0
625
626
627class SetPgid(ChildStateChange):
628
629 def __init__(self, pgid, tracer):
630 # type: (int, dev.Tracer) -> None
631 self.pgid = pgid
632 self.tracer = tracer
633
634 def Apply(self):
635 # type: () -> None
636 try:
637 posix.setpgid(0, self.pgid)
638 except (IOError, OSError) as e:
639 self.tracer.OtherMessage(
640 'osh: child %d failed to set its process group to %d: %s' %
641 (posix.getpid(), self.pgid, pyutil.strerror(e)))
642
643 def ApplyFromParent(self, proc):
644 # type: (Process) -> None
645 try:
646 posix.setpgid(proc.pid, self.pgid)
647 except (IOError, OSError) as e:
648 self.tracer.OtherMessage(
649 'osh: parent failed to set process group for PID %d to %d: %s'
650 % (proc.pid, self.pgid, pyutil.strerror(e)))
651
652
653class ExternalProgram(object):
654 """The capability to execute an external program like 'ls'."""
655
656 def __init__(
657 self,
658 hijack_shebang, # type: str
659 fd_state, # type: FdState
660 errfmt, # type: ui.ErrorFormatter
661 debug_f, # type: _DebugFile
662 ):
663 # type: (...) -> None
664 """
665 Args:
666 hijack_shebang: The path of an interpreter to run instead of the one
667 specified in the shebang line. May be empty.
668 """
669 self.hijack_shebang = hijack_shebang
670 self.fd_state = fd_state
671 self.errfmt = errfmt
672 self.debug_f = debug_f
673
674 def Exec(self, argv0_path, cmd_val, environ):
675 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
676 """Execute a program and exit this process.
677
678 Called by: ls / exec ls / ( ls / )
679 """
680 probe('process', 'ExternalProgram_Exec', argv0_path)
681 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
682 True)
683 assert False, "This line should never execute" # NO RETURN
684
685 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
686 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
687 if len(self.hijack_shebang):
688 opened = True
689 try:
690 f = self.fd_state.Open(argv0_path)
691 except (IOError, OSError) as e:
692 opened = False
693
694 if opened:
695 with ctx_FileCloser(f):
696 # Test if the shebang looks like a shell. TODO: The file might be
697 # binary with no newlines, so read 80 bytes instead of readline().
698
699 #line = f.read(80) # type: ignore # TODO: fix this
700 line = f.readline()
701
702 if match.ShouldHijack(line):
703 h_argv = [self.hijack_shebang, argv0_path]
704 h_argv.extend(argv[1:])
705 argv = h_argv
706 argv0_path = self.hijack_shebang
707 self.debug_f.writeln('Hijacked: %s' % argv0_path)
708 else:
709 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
710 pass
711
712 try:
713 posix.execve(argv0_path, argv, environ)
714 except (IOError, OSError) as e:
715 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
716 if e.errno == ENOEXEC and should_retry:
717 new_argv = ['/bin/sh', argv0_path]
718 new_argv.extend(argv[1:])
719 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
720 # NO RETURN
721
722 # Would be nice: when the path is relative and ENOENT: print PWD and do
723 # spelling correction?
724
725 self.errfmt.Print_(
726 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
727 argv0_loc)
728
729 # POSIX mentions 126 and 127 for two specific errors. The rest are
730 # unspecified.
731 #
732 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
733 if e.errno == EACCES:
734 status = 126
735 elif e.errno == ENOENT:
736 # TODO: most shells print 'command not found', rather than strerror()
737 # == "No such file or directory". That's better because it's at the
738 # end of the path search, and we're never searching for a directory.
739 status = 127
740 else:
741 # dash uses 2, but we use that for parse errors. This seems to be
742 # consistent with mksh and zsh.
743 status = 127
744
745 posix._exit(status)
746 # NO RETURN
747
748
749class Thunk(object):
750 """Abstract base class for things runnable in another process."""
751
752 def __init__(self):
753 # type: () -> None
754 """Empty constructor for mycpp."""
755 pass
756
757 def Run(self):
758 # type: () -> None
759 """Returns a status code."""
760 raise NotImplementedError()
761
762 def UserString(self):
763 # type: () -> str
764 """Display for the 'jobs' list."""
765 raise NotImplementedError()
766
767 def __repr__(self):
768 # type: () -> str
769 return self.UserString()
770
771
772class ExternalThunk(Thunk):
773 """An external executable."""
774
775 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
776 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
777 self.ext_prog = ext_prog
778 self.argv0_path = argv0_path
779 self.cmd_val = cmd_val
780 self.environ = environ
781
782 def UserString(self):
783 # type: () -> str
784
785 # NOTE: This is the format the Tracer uses.
786 # bash displays sleep $n & (code)
787 # but OSH displays sleep 1 & (argv array)
788 # We could switch the former but I'm not sure it's necessary.
789 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
790 return '[process] %s' % ' '.join(tmp)
791
792 def Run(self):
793 # type: () -> None
794 """An ExternalThunk is run in parent for the exec builtin."""
795 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
796
797
798class SubProgramThunk(Thunk):
799 """A subprogram that can be executed in another process."""
800
801 def __init__(self,
802 cmd_ev,
803 node,
804 trap_state,
805 multi_trace,
806 inherit_errexit=True):
807 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool) -> None
808 self.cmd_ev = cmd_ev
809 self.node = node
810 self.trap_state = trap_state
811 self.multi_trace = multi_trace
812 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
813
814 def UserString(self):
815 # type: () -> str
816
817 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
818 # TODO: Extract SPIDS from node to display source? Note that
819 # CompoundStatus also has locations of each pipeline component; see
820 # Executor.RunPipeline()
821 thunk_str = ui.CommandType(self.node)
822 return '[subprog] %s' % thunk_str
823
824 def Run(self):
825 # type: () -> None
826 #self.errfmt.OneLineErrExit() # don't quote code in child processes
827 probe('process', 'SubProgramThunk_Run')
828
829 # TODO: break circular dep. Bit flags could go in ASDL or headers.
830 from osh import cmd_eval
831
832 # signal handlers aren't inherited
833 self.trap_state.ClearForSubProgram()
834
835 # NOTE: may NOT return due to exec().
836 if not self.inherit_errexit:
837 self.cmd_ev.mutable_opts.DisableErrExit()
838 try:
839 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
840 self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
841 status = self.cmd_ev.LastStatus()
842 # NOTE: We ignore the is_fatal return value. The user should set -o
843 # errexit so failures in subprocesses cause failures in the parent.
844 except util.UserExit as e:
845 status = e.status
846
847 # Handle errors in a subshell. These two cases are repeated from main()
848 # and the core/completion.py hook.
849 except KeyboardInterrupt:
850 print('')
851 status = 130 # 128 + 2
852 except (IOError, OSError) as e:
853 print_stderr('oils I/O error (subprogram): %s' %
854 pyutil.strerror(e))
855 status = 2
856
857 # If ProcessInit() doesn't turn off buffering, this is needed before
858 # _exit()
859 pyos.FlushStdout()
860
861 self.multi_trace.WriteDumps()
862
863 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
864 # gets called in BOTH processes.
865 # The crash dump seems to be unaffected.
866 posix._exit(status)
867
868
869class _HereDocWriterThunk(Thunk):
870 """Write a here doc to one end of a pipe.
871
872 May be be executed in either a child process or the main shell
873 process.
874 """
875
876 def __init__(self, w, body_str):
877 # type: (int, str) -> None
878 self.w = w
879 self.body_str = body_str
880
881 def UserString(self):
882 # type: () -> str
883
884 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
885 # shells don't have this problem because they use temp files! That's a bit
886 # unfortunate.
887 return '[here doc writer]'
888
889 def Run(self):
890 # type: () -> None
891 """do_exit: For small pipelines."""
892 probe('process', 'HereDocWriterThunk_Run')
893 #log('Writing %r', self.body_str)
894 posix.write(self.w, self.body_str)
895 #log('Wrote %r', self.body_str)
896 posix.close(self.w)
897 #log('Closed %d', self.w)
898
899 posix._exit(0)
900
901
902class Job(object):
903 """Interface for both Process and Pipeline.
904
905 They both can be put in the background and waited on.
906
907 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
908
909 sleep 1 | sleep 2 &
910
911 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
912 not a JOB ID.
913 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
914 - The process group leader (setpgid) is the FIRST PID.
915 - It's also %1 or %+. The last job started.
916 """
917
918 def __init__(self):
919 # type: () -> None
920 # Initial state with & or Ctrl-Z is Running.
921 self.state = job_state_e.Running
922 self.job_id = -1
923 self.in_background = False
924
925 def DisplayJob(self, job_id, f, style):
926 # type: (int, mylib.Writer, int) -> None
927 raise NotImplementedError()
928
929 def State(self):
930 # type: () -> job_state_t
931 return self.state
932
933 def ProcessGroupId(self):
934 # type: () -> int
935 """Return the process group ID associated with this job."""
936 raise NotImplementedError()
937
938 def JobWait(self, waiter):
939 # type: (Waiter) -> wait_status_t
940 """Wait for this process/pipeline to be stopped or finished."""
941 raise NotImplementedError()
942
943 def SetBackground(self):
944 # type: () -> None
945 """Record that this job is running in the background."""
946 self.in_background = True
947
948 def SetForeground(self):
949 # type: () -> None
950 """Record that this job is running in the foreground."""
951 self.in_background = False
952
953
954class Process(Job):
955 """A process to run.
956
957 TODO: Should we make it clear that this is a FOREGROUND process? A
958 background process is wrapped in a "job". It is unevaluated.
959
960 It provides an API to manipulate file descriptor state in parent and child.
961 """
962
963 def __init__(self, thunk, job_control, job_list, tracer):
964 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
965 """
966 Args:
967 thunk: Thunk instance
968 job_list: for process bookkeeping
969 """
970 Job.__init__(self)
971 assert isinstance(thunk, Thunk), thunk
972 self.thunk = thunk
973 self.job_control = job_control
974 self.job_list = job_list
975 self.tracer = tracer
976
977 # For pipelines
978 self.parent_pipeline = None # type: Pipeline
979 self.state_changes = [] # type: List[ChildStateChange]
980 self.close_r = -1
981 self.close_w = -1
982
983 self.pid = -1
984 self.status = -1
985
986 def Init_ParentPipeline(self, pi):
987 # type: (Pipeline) -> None
988 """For updating PIPESTATUS."""
989 self.parent_pipeline = pi
990
991 def __repr__(self):
992 # type: () -> str
993
994 # note: be wary of infinite mutual recursion
995 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
996 #return '<Process %s%s>' % (self.thunk, s)
997 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
998
999 def ProcessGroupId(self):
1000 # type: () -> int
1001 """Returns the group ID of this process."""
1002 # This should only ever be called AFTER the process has started
1003 assert self.pid != -1
1004 if self.parent_pipeline:
1005 # XXX: Maybe we should die here instead? Unclear if this branch
1006 # should even be reachable with the current builtins.
1007 return self.parent_pipeline.ProcessGroupId()
1008
1009 return self.pid
1010
1011 def DisplayJob(self, job_id, f, style):
1012 # type: (int, mylib.Writer, int) -> None
1013 if job_id == -1:
1014 job_id_str = ' '
1015 else:
1016 job_id_str = '%%%d' % job_id
1017 if style == STYLE_PID_ONLY:
1018 f.write('%d\n' % self.pid)
1019 else:
1020 f.write('%s %d %7s ' %
1021 (job_id_str, self.pid, _JobStateStr(self.state)))
1022 f.write(self.thunk.UserString())
1023 f.write('\n')
1024
1025 def AddStateChange(self, s):
1026 # type: (ChildStateChange) -> None
1027 self.state_changes.append(s)
1028
1029 def AddPipeToClose(self, r, w):
1030 # type: (int, int) -> None
1031 self.close_r = r
1032 self.close_w = w
1033
1034 def MaybeClosePipe(self):
1035 # type: () -> None
1036 if self.close_r != -1:
1037 posix.close(self.close_r)
1038 posix.close(self.close_w)
1039
1040 def StartProcess(self, why):
1041 # type: (trace_t) -> int
1042 """Start this process with fork(), handling redirects."""
1043 pid = posix.fork()
1044 if pid < 0:
1045 # When does this happen?
1046 e_die('Fatal error in posix.fork()')
1047
1048 elif pid == 0: # child
1049 # Note: this happens in BOTH interactive and non-interactive shells.
1050 # We technically don't need to do most of it in non-interactive, since we
1051 # did not change state in InitInteractiveShell().
1052
1053 for st in self.state_changes:
1054 st.Apply()
1055
1056 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1057 # shouldn't have this.
1058 # https://docs.python.org/2/library/signal.html
1059 # See Python/pythonrun.c.
1060 pyos.Sigaction(SIGPIPE, SIG_DFL)
1061
1062 # Respond to Ctrl-\ (core dump)
1063 pyos.Sigaction(SIGQUIT, SIG_DFL)
1064
1065 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1066 # foreground because suspending them is difficult with our 'lastpipe'
1067 # semantics.
1068 pid = posix.getpid()
1069 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1070 pyos.Sigaction(SIGTSTP, SIG_DFL)
1071
1072 # More signals from
1073 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1074 # (but not SIGCHLD)
1075 pyos.Sigaction(SIGTTOU, SIG_DFL)
1076 pyos.Sigaction(SIGTTIN, SIG_DFL)
1077
1078 self.tracer.OnNewProcess(pid)
1079 # clear foreground pipeline for subshells
1080 self.thunk.Run()
1081 # Never returns
1082
1083 #log('STARTED process %s, pid = %d', self, pid)
1084 self.tracer.OnProcessStart(pid, why)
1085
1086 # Class invariant: after the process is started, it stores its PID.
1087 self.pid = pid
1088
1089 # SetPgid needs to be applied from the child and the parent to avoid
1090 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1091 for st in self.state_changes:
1092 st.ApplyFromParent(self)
1093
1094 # Program invariant: We keep track of every child process!
1095 self.job_list.AddChildProcess(pid, self)
1096
1097 return pid
1098
1099 def Wait(self, waiter):
1100 # type: (Waiter) -> int
1101 """Wait for this process to finish."""
1102 while self.state == job_state_e.Running:
1103 # Only return if there's nothing to wait for. Keep waiting if we were
1104 # interrupted with a signal.
1105 if waiter.WaitForOne() == W1_ECHILD:
1106 break
1107
1108 assert self.status >= 0, self.status
1109 return self.status
1110
1111 def JobWait(self, waiter):
1112 # type: (Waiter) -> wait_status_t
1113 # wait builtin can be interrupted
1114 while self.state == job_state_e.Running:
1115 result = waiter.WaitForOne()
1116
1117 if result >= 0: # signal
1118 return wait_status.Cancelled(result)
1119
1120 if result == W1_ECHILD:
1121 break
1122
1123 return wait_status.Proc(self.status)
1124
1125 def WhenStopped(self, stop_sig):
1126 # type: (int) -> None
1127
1128 # 128 is a shell thing
1129 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1130 self.status = 128 + stop_sig
1131 self.state = job_state_e.Stopped
1132
1133 if self.job_id == -1:
1134 # This process was started in the foreground
1135 self.job_list.AddJob(self)
1136
1137 if not self.in_background:
1138 self.job_control.MaybeTakeTerminal()
1139 self.SetBackground()
1140
1141 def WhenDone(self, pid, status):
1142 # type: (int, int) -> None
1143 """Called by the Waiter when this Process finishes."""
1144
1145 #log('WhenDone %d %d', pid, status)
1146 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1147 self.status = status
1148 self.state = job_state_e.Done
1149 if self.parent_pipeline:
1150 self.parent_pipeline.WhenDone(pid, status)
1151 else:
1152 if self.job_id != -1:
1153 # Job might have been brought to the foreground after being
1154 # assigned a job ID.
1155 if self.in_background:
1156 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1157
1158 self.job_list.RemoveJob(self.job_id)
1159
1160 self.job_list.RemoveChildProcess(self.pid)
1161
1162 if not self.in_background:
1163 self.job_control.MaybeTakeTerminal()
1164
1165 def RunProcess(self, waiter, why):
1166 # type: (Waiter, trace_t) -> int
1167 """Run this process synchronously."""
1168 self.StartProcess(why)
1169 # ShellExecutor might be calling this for the last part of a pipeline.
1170 if self.parent_pipeline is None:
1171 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1172 # calling getpgid()?
1173 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1174 return self.Wait(waiter)
1175
1176
1177class ctx_Pipe(object):
1178
1179 def __init__(self, fd_state, fd, err_out):
1180 # type: (FdState, int, List[error.IOError_OSError]) -> None
1181 fd_state.PushStdinFromPipe(fd)
1182 self.fd_state = fd_state
1183 self.err_out = err_out
1184
1185 def __enter__(self):
1186 # type: () -> None
1187 pass
1188
1189 def __exit__(self, type, value, traceback):
1190 # type: (Any, Any, Any) -> None
1191 self.fd_state.Pop(self.err_out)
1192
1193
1194class Pipeline(Job):
1195 """A pipeline of processes to run.
1196
1197 Cases we handle:
1198
1199 foo | bar
1200 $(foo | bar)
1201 foo | bar | read v
1202 """
1203
1204 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1205 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1206 Job.__init__(self)
1207 self.job_control = job_control
1208 self.job_list = job_list
1209 self.tracer = tracer
1210
1211 self.procs = [] # type: List[Process]
1212 self.pids = [] # type: List[int] # pids in order
1213 self.pipe_status = [] # type: List[int] # status in order
1214 self.status = -1 # for 'wait' jobs
1215
1216 self.pgid = INVALID_PGID
1217
1218 # Optional for foreground
1219 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1220 self.last_pipe = None # type: Tuple[int, int]
1221
1222 self.sigpipe_status_ok = sigpipe_status_ok
1223
1224 def ProcessGroupId(self):
1225 # type: () -> int
1226 """Returns the group ID of this pipeline."""
1227 return self.pgid
1228
1229 def DisplayJob(self, job_id, f, style):
1230 # type: (int, mylib.Writer, int) -> None
1231 if style == STYLE_PID_ONLY:
1232 f.write('%d\n' % self.procs[0].pid)
1233 else:
1234 # Note: this is STYLE_LONG.
1235 for i, proc in enumerate(self.procs):
1236 if i == 0: # show job ID for first element in pipeline
1237 job_id_str = '%%%d' % job_id
1238 else:
1239 job_id_str = ' ' # 2 spaces
1240
1241 f.write('%s %d %7s ' %
1242 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1243 f.write(proc.thunk.UserString())
1244 f.write('\n')
1245
1246 def DebugPrint(self):
1247 # type: () -> None
1248 print('Pipeline in state %s' % _JobStateStr(self.state))
1249 if mylib.PYTHON: # %s for Process not allowed in C++
1250 for proc in self.procs:
1251 print(' proc %s' % proc)
1252 _, last_node = self.last_thunk
1253 print(' last %s' % last_node)
1254 print(' pipe_status %s' % self.pipe_status)
1255
1256 def Add(self, p):
1257 # type: (Process) -> None
1258 """Append a process to the pipeline."""
1259 if len(self.procs) == 0:
1260 self.procs.append(p)
1261 return
1262
1263 r, w = posix.pipe()
1264 #log('pipe for %s: %d %d', p, r, w)
1265 prev = self.procs[-1]
1266
1267 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1268 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1269
1270 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1271
1272 self.procs.append(p)
1273
1274 def AddLast(self, thunk):
1275 # type: (Tuple[CommandEvaluator, command_t]) -> None
1276 """Append the last noden to the pipeline.
1277
1278 This is run in the CURRENT process. It is OPTIONAL, because
1279 pipelines in the background are run uniformly.
1280 """
1281 self.last_thunk = thunk
1282
1283 assert len(self.procs) != 0
1284
1285 r, w = posix.pipe()
1286 prev = self.procs[-1]
1287 prev.AddStateChange(StdoutToPipe(r, w))
1288
1289 self.last_pipe = (r, w) # So we can connect it to last_thunk
1290
1291 def StartPipeline(self, waiter):
1292 # type: (Waiter) -> None
1293
1294 # If we are creating a pipeline in a subshell or we aren't running with job
1295 # control, our children should remain in our inherited process group.
1296 # the pipelines's group ID.
1297 if self.job_control.Enabled():
1298 self.pgid = OWN_LEADER # first process in pipeline is the leader
1299
1300 for i, proc in enumerate(self.procs):
1301 if self.pgid != INVALID_PGID:
1302 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1303
1304 # Figure out the pid
1305 pid = proc.StartProcess(trace.PipelinePart)
1306 if i == 0 and self.pgid != INVALID_PGID:
1307 # Mimic bash and use the PID of the FIRST process as the group for the
1308 # whole pipeline.
1309 self.pgid = pid
1310
1311 self.pids.append(pid)
1312 self.pipe_status.append(-1) # uninitialized
1313
1314 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1315 # It can't be done at the end; otherwise processes will have descriptors
1316 # from non-adjacent pipes.
1317 proc.MaybeClosePipe()
1318
1319 if self.last_thunk:
1320 self.pipe_status.append(-1) # for self.last_thunk
1321
1322 def LastPid(self):
1323 # type: () -> int
1324 """For the odd $! variable.
1325
1326 It would be better if job IDs or PGIDs were used consistently.
1327 """
1328 return self.pids[-1]
1329
1330 def Wait(self, waiter):
1331 # type: (Waiter) -> List[int]
1332 """Wait for this pipeline to finish."""
1333
1334 assert self.procs, "no procs for Wait()"
1335 # waitpid(-1) zero or more times
1336 while self.state == job_state_e.Running:
1337 # Keep waiting until there's nothing to wait for.
1338 if waiter.WaitForOne() == W1_ECHILD:
1339 break
1340
1341 return self.pipe_status
1342
1343 def JobWait(self, waiter):
1344 # type: (Waiter) -> wait_status_t
1345 """Called by 'wait' builtin, e.g. 'wait %1'."""
1346 # wait builtin can be interrupted
1347 assert self.procs, "no procs for Wait()"
1348 while self.state == job_state_e.Running:
1349 result = waiter.WaitForOne()
1350
1351 if result >= 0: # signal
1352 return wait_status.Cancelled(result)
1353
1354 if result == W1_ECHILD:
1355 break
1356
1357 return wait_status.Pipeline(self.pipe_status)
1358
1359 def RunLastPart(self, waiter, fd_state):
1360 # type: (Waiter, FdState) -> List[int]
1361 """Run this pipeline synchronously (foreground pipeline).
1362
1363 Returns:
1364 pipe_status (list of integers).
1365 """
1366 assert len(self.pids) == len(self.procs)
1367
1368 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1369 from osh import cmd_eval
1370
1371 # This is tcsetpgrp()
1372 # TODO: fix race condition -- I believe the first process could have
1373 # stopped already, and thus getpgid() will fail
1374 self.job_control.MaybeGiveTerminal(self.pgid)
1375
1376 # Run the last part of the pipeline IN PARALLEL with other processes. It
1377 # may or may not fork:
1378 # echo foo | read line # no fork, the builtin runs in THIS shell process
1379 # ls | wc -l # fork for 'wc'
1380
1381 cmd_ev, last_node = self.last_thunk
1382
1383 assert self.last_pipe is not None
1384 r, w = self.last_pipe # set in AddLast()
1385 posix.close(w) # we will not write here
1386
1387 # Fix lastpipe / job control / DEBUG trap interaction
1388 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1389
1390 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1391 # a pipeline.
1392 cmd_flags |= cmd_eval.NoErrTrap
1393
1394 io_errors = [] # type: List[error.IOError_OSError]
1395 with ctx_Pipe(fd_state, r, io_errors):
1396 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1397
1398 if len(io_errors):
1399 e_die('Error setting up last part of pipeline: %s' %
1400 pyutil.strerror(io_errors[0]))
1401
1402 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1403 # /dev/urandom | sleep 1' will never get SIGPIPE.
1404 posix.close(r)
1405
1406 self.pipe_status[-1] = cmd_ev.LastStatus()
1407 if self.AllDone():
1408 self.state = job_state_e.Done
1409
1410 #log('pipestatus before all have finished = %s', self.pipe_status)
1411 return self.Wait(waiter)
1412
1413 def AllDone(self):
1414 # type: () -> bool
1415
1416 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1417 for status in self.pipe_status:
1418 if status == -1:
1419 return False
1420 return True
1421
1422 def WhenDone(self, pid, status):
1423 # type: (int, int) -> None
1424 """Called by Process.WhenDone."""
1425 #log('Pipeline WhenDone %d %d', pid, status)
1426 i = self.pids.index(pid)
1427 assert i != -1, 'Unexpected PID %d' % pid
1428
1429 if status == 141 and self.sigpipe_status_ok:
1430 status = 0
1431
1432 self.job_list.RemoveChildProcess(pid)
1433 self.pipe_status[i] = status
1434 if self.AllDone():
1435 if self.job_id != -1:
1436 # Job might have been brought to the foreground after being
1437 # assigned a job ID.
1438 if self.in_background:
1439 print_stderr('[%d] Done PGID %d' %
1440 (self.job_id, self.pids[0]))
1441
1442 self.job_list.RemoveJob(self.job_id)
1443
1444 # status of pipeline is status of last process
1445 self.status = self.pipe_status[-1]
1446 self.state = job_state_e.Done
1447 if not self.in_background:
1448 self.job_control.MaybeTakeTerminal()
1449
1450
1451def _JobStateStr(i):
1452 # type: (job_state_t) -> str
1453 return job_state_str(i)[10:] # remove 'job_state.'
1454
1455
1456def _GetTtyFd():
1457 # type: () -> int
1458 """Returns -1 if stdio is not a TTY."""
1459 try:
1460 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1461 except (IOError, OSError) as e:
1462 return -1
1463
1464
1465class ctx_TerminalControl(object):
1466
1467 def __init__(self, job_control, errfmt):
1468 # type: (JobControl, ui.ErrorFormatter) -> None
1469 job_control.InitJobControl()
1470 self.job_control = job_control
1471 self.errfmt = errfmt
1472
1473 def __enter__(self):
1474 # type: () -> None
1475 pass
1476
1477 def __exit__(self, type, value, traceback):
1478 # type: (Any, Any, Any) -> None
1479
1480 # Return the TTY to the original owner before exiting.
1481 try:
1482 self.job_control.MaybeReturnTerminal()
1483 except error.FatalRuntime as e:
1484 # Don't abort the shell on error, just print a message.
1485 self.errfmt.PrettyPrintError(e)
1486
1487
1488class JobControl(object):
1489 """Interface to setpgid(), tcsetpgrp(), etc."""
1490
1491 def __init__(self):
1492 # type: () -> None
1493
1494 # The main shell's PID and group ID.
1495 self.shell_pid = -1
1496 self.shell_pgid = -1
1497
1498 # The fd of the controlling tty. Set to -1 when job control is disabled.
1499 self.shell_tty_fd = -1
1500
1501 # For giving the terminal back to our parent before exiting (if not a login
1502 # shell).
1503 self.original_tty_pgid = -1
1504
1505 def InitJobControl(self):
1506 # type: () -> None
1507 self.shell_pid = posix.getpid()
1508 orig_shell_pgid = posix.getpgid(0)
1509 self.shell_pgid = orig_shell_pgid
1510 self.shell_tty_fd = _GetTtyFd()
1511
1512 # If we aren't the leader of our process group, create a group and mark
1513 # ourselves as the leader.
1514 if self.shell_pgid != self.shell_pid:
1515 try:
1516 posix.setpgid(self.shell_pid, self.shell_pid)
1517 self.shell_pgid = self.shell_pid
1518 except (IOError, OSError) as e:
1519 self.shell_tty_fd = -1
1520
1521 if self.shell_tty_fd != -1:
1522 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1523
1524 # If stdio is a TTY, put the shell's process group in the foreground.
1525 try:
1526 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1527 except (IOError, OSError) as e:
1528 # We probably aren't in the session leader's process group. Disable job
1529 # control.
1530 self.shell_tty_fd = -1
1531 self.shell_pgid = orig_shell_pgid
1532 posix.setpgid(self.shell_pid, self.shell_pgid)
1533
1534 def Enabled(self):
1535 # type: () -> bool
1536
1537 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1538 # think.
1539 curr_pid = posix.getpid()
1540 # Only the main shell should bother with job control functions.
1541 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1542
1543 # TODO: This isn't a PID. This is a process group ID?
1544 #
1545 # What should the table look like?
1546 #
1547 # Do we need the last PID? I don't know why bash prints that. Probably so
1548 # you can do wait $!
1549 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1550 #
1551 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1552 # job_id is just an integer. This is sort of lame.
1553 #
1554 # [job_id, flag, pgid, job_state, node]
1555
1556 def MaybeGiveTerminal(self, pgid):
1557 # type: (int) -> None
1558 """If stdio is a TTY, move the given process group to the
1559 foreground."""
1560 if not self.Enabled():
1561 # Only call tcsetpgrp when job control is enabled.
1562 return
1563
1564 try:
1565 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1566 except (IOError, OSError) as e:
1567 e_die('osh: Failed to move process group %d to foreground: %s' %
1568 (pgid, pyutil.strerror(e)))
1569
1570 def MaybeTakeTerminal(self):
1571 # type: () -> None
1572 """If stdio is a TTY, return the main shell's process group to the
1573 foreground."""
1574 self.MaybeGiveTerminal(self.shell_pgid)
1575
1576 def MaybeReturnTerminal(self):
1577 # type: () -> None
1578 """Called before the shell exits."""
1579 self.MaybeGiveTerminal(self.original_tty_pgid)
1580
1581
1582class JobList(object):
1583 """Global list of jobs, used by a few builtins."""
1584
1585 def __init__(self):
1586 # type: () -> None
1587
1588 # job_id -> Job instance
1589 self.jobs = {} # type: Dict[int, Job]
1590
1591 # pid -> Process. This is for STOP notification.
1592 self.child_procs = {} # type: Dict[int, Process]
1593 self.debug_pipelines = [] # type: List[Pipeline]
1594
1595 # Counter used to assign IDs to jobs. It is incremented every time a job
1596 # is created. Once all active jobs are done it is reset to 1. I'm not
1597 # sure if this reset behavior is mandated by POSIX, but other shells do
1598 # it, so we mimic for the sake of compatibility.
1599 self.job_id = 1
1600
1601 def AddJob(self, job):
1602 # type: (Job) -> int
1603 """Add a background job to the list.
1604
1605 A job is either a Process or Pipeline. You can resume a job with 'fg',
1606 kill it with 'kill', etc.
1607
1608 Two cases:
1609
1610 1. async jobs: sleep 5 | sleep 4 &
1611 2. stopped jobs: sleep 5; then Ctrl-Z
1612 """
1613 job_id = self.job_id
1614 self.jobs[job_id] = job
1615 job.job_id = job_id
1616 self.job_id += 1
1617 return job_id
1618
1619 def RemoveJob(self, job_id):
1620 # type: (int) -> None
1621 """Process and Pipeline can call this."""
1622 mylib.dict_erase(self.jobs, job_id)
1623
1624 if len(self.jobs) == 0:
1625 self.job_id = 1
1626
1627 def AddChildProcess(self, pid, proc):
1628 # type: (int, Process) -> None
1629 """Every child process should be added here as soon as we know its PID.
1630
1631 When the Waiter gets an EXITED or STOPPED notification, we need
1632 to know about it so 'jobs' can work.
1633 """
1634 self.child_procs[pid] = proc
1635
1636 def RemoveChildProcess(self, pid):
1637 # type: (int) -> None
1638 """Remove the child process with the given PID."""
1639 mylib.dict_erase(self.child_procs, pid)
1640
1641 if mylib.PYTHON:
1642
1643 def AddPipeline(self, pi):
1644 # type: (Pipeline) -> None
1645 """For debugging only."""
1646 self.debug_pipelines.append(pi)
1647
1648 def ProcessFromPid(self, pid):
1649 # type: (int) -> Process
1650 """For wait $PID.
1651
1652 There's no way to wait for a pipeline with a PID. That uses job
1653 syntax, e.g. %1. Not a great interface.
1654 """
1655 return self.child_procs.get(pid)
1656
1657 def GetCurrentAndPreviousJobs(self):
1658 # type: () -> Tuple[Optional[Job], Optional[Job]]
1659 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1660
1661 See the POSIX specification for the `jobs` builtin for details:
1662 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1663
1664 IMPORTANT NOTE: This method assumes that the jobs list will not change
1665 during its execution! This assumption holds for now because we only ever
1666 update the jobs list from the main loop after WaitPid() informs us of a
1667 change. If we implement `set -b` and install a signal handler for
1668 SIGCHLD we should be careful to synchronize it with this function. The
1669 unsafety of mutating GC data structures from a signal handler should
1670 make this a non-issue, but if bugs related to this appear this note may
1671 be helpful...
1672 """
1673 # Split all active jobs by state and sort each group by decreasing job
1674 # ID to approximate newness.
1675 stopped_jobs = [] # type: List[Job]
1676 running_jobs = [] # type: List[Job]
1677 for i in xrange(0, self.job_id):
1678 job = self.jobs.get(i, None)
1679 if not job:
1680 continue
1681
1682 if job.state == job_state_e.Stopped:
1683 stopped_jobs.append(job)
1684
1685 elif job.state == job_state_e.Running:
1686 running_jobs.append(job)
1687
1688 current = None # type: Optional[Job]
1689 previous = None # type: Optional[Job]
1690 # POSIX says: If there is any suspended job, then the current job shall
1691 # be a suspended job. If there are at least two suspended jobs, then the
1692 # previous job also shall be a suspended job.
1693 #
1694 # So, we will only return running jobs from here if there are no recent
1695 # stopped jobs.
1696 if len(stopped_jobs) > 0:
1697 current = stopped_jobs.pop()
1698
1699 if len(stopped_jobs) > 0:
1700 previous = stopped_jobs.pop()
1701
1702 if len(running_jobs) > 0 and not current:
1703 current = running_jobs.pop()
1704
1705 if len(running_jobs) > 0 and not previous:
1706 previous = running_jobs.pop()
1707
1708 if not previous:
1709 previous = current
1710
1711 return current, previous
1712
1713 def GetJobWithSpec(self, job_spec):
1714 # type: (str) -> Optional[Job]
1715 """Parse the given job spec and return the matching job. If there is no
1716 matching job, this function returns None.
1717
1718 See the POSIX spec for the `jobs` builtin for details about job specs:
1719 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1720 """
1721 if job_spec in CURRENT_JOB_SPECS:
1722 current, _ = self.GetCurrentAndPreviousJobs()
1723 return current
1724
1725 if job_spec == '%-':
1726 _, previous = self.GetCurrentAndPreviousJobs()
1727 return previous
1728
1729 # TODO: Add support for job specs based on prefixes of process argv.
1730 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1731 if m is not None:
1732 assert len(m) == 2
1733 job_id = int(m[1])
1734 if job_id in self.jobs:
1735 return self.jobs[job_id]
1736
1737 return None
1738
1739 def DisplayJobs(self, style):
1740 # type: (int) -> None
1741 """Used by the 'jobs' builtin.
1742
1743 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1744
1745 "By default, the jobs utility shall display the status of all stopped jobs,
1746 running background jobs and all jobs whose status has changed and have not
1747 been reported by the shell."
1748 """
1749 # NOTE: A job is a background process or pipeline.
1750 #
1751 # echo hi | wc -l -- this starts two processes. Wait for TWO
1752 # echo hi | wc -l & -- this starts a process which starts two processes
1753 # Wait for ONE.
1754 #
1755 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1756 # for %% and %-, PID, status, and "command".
1757 #
1758 # Every component of a pipeline is on the same line with 'jobs', but
1759 # they're separated into different lines with 'jobs -l'.
1760 #
1761 # See demo/jobs-builtin.sh
1762
1763 # $ jobs -l
1764 # [1]+ 24414 Stopped sleep 5
1765 # 24415 | sleep 5
1766 # [2] 24502 Running sleep 6
1767 # 24503 | sleep 6
1768 # 24504 | sleep 5 &
1769 # [3]- 24508 Running sleep 6
1770 # 24509 | sleep 6
1771 # 24510 | sleep 5 &
1772
1773 f = mylib.Stdout()
1774 for job_id, job in iteritems(self.jobs):
1775 # Use the %1 syntax
1776 job.DisplayJob(job_id, f, style)
1777
1778 def DebugPrint(self):
1779 # type: () -> None
1780
1781 f = mylib.Stdout()
1782 f.write('\n')
1783 f.write('[process debug info]\n')
1784
1785 for pid, proc in iteritems(self.child_procs):
1786 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1787 #p = ' |' if proc.parent_pipeline else ''
1788 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1789
1790 if len(self.debug_pipelines):
1791 f.write('\n')
1792 f.write('[pipeline debug info]\n')
1793 for pi in self.debug_pipelines:
1794 pi.DebugPrint()
1795
1796 def ListRecent(self):
1797 # type: () -> None
1798 """For jobs -n, which I think is also used in the interactive
1799 prompt."""
1800 pass
1801
1802 def NumRunning(self):
1803 # type: () -> int
1804 """Return the number of running jobs.
1805
1806 Used by 'wait' and 'wait -n'.
1807 """
1808 count = 0
1809 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1810 if job.State() == job_state_e.Running:
1811 count += 1
1812 return count
1813
1814
1815# Some WaitForOne() return values
1816W1_OK = -2 # waitpid(-1) returned
1817W1_ECHILD = -3 # no processes to wait for
1818W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1819
1820
1821class Waiter(object):
1822 """A capability to wait for processes.
1823
1824 This must be a singleton (and is because CommandEvaluator is a singleton).
1825
1826 Invariants:
1827 - Every child process is registered once
1828 - Every child process is waited for
1829
1830 Canonical example of why we need a GLOBAL waiter:
1831
1832 { sleep 3; echo 'done 3'; } &
1833 { sleep 4; echo 'done 4'; } &
1834
1835 # ... do arbitrary stuff ...
1836
1837 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1838
1839 Now when you do wait() after starting the pipeline, you might get a pipeline
1840 process OR a background process! So you have to distinguish between them.
1841 """
1842
1843 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1844 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1845 self.job_list = job_list
1846 self.exec_opts = exec_opts
1847 self.signal_safe = signal_safe
1848 self.tracer = tracer
1849 self.last_status = 127 # wait -n error code
1850
1851 def WaitForOne(self, waitpid_options=0):
1852 # type: (int) -> int
1853 """Wait until the next process returns (or maybe Ctrl-C).
1854
1855 Returns:
1856 One of these negative numbers:
1857 W1_ECHILD Nothing to wait for
1858 W1_OK Caller should keep waiting
1859 UNTRAPPED_SIGWINCH
1860 Or
1861 result > 0 Signal that waitpid() was interrupted with
1862
1863 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1864 will try again.
1865
1866 Callers:
1867 wait -n -- loop until there is one fewer process (TODO)
1868 wait -- loop until there are no processes
1869 wait $! -- loop until job state is Done (process or pipeline)
1870 Process::Wait() -- loop until Process state is done
1871 Pipeline::Wait() -- loop until Pipeline state is done
1872
1873 Comparisons:
1874 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1875 the wait builtin
1876
1877 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1878 loop while (gotsigchld), but that might be a hack for System V!
1879
1880 Should we have a cleaner API like named posix::wait_for_one() ?
1881
1882 wait_result =
1883 ECHILD -- nothing to wait for
1884 | Done(int pid, int status) -- process done
1885 | EINTR(bool sigint) -- may or may not retry
1886 """
1887 pid, status = pyos.WaitPid(waitpid_options)
1888 if pid == 0: # WNOHANG passed, and no state changes
1889 return W1_AGAIN
1890 elif pid < 0: # error case
1891 err_num = status
1892 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1893 if err_num == ECHILD:
1894 return W1_ECHILD # nothing to wait for caller should stop
1895 elif err_num == EINTR: # Bug #858 fix
1896 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1897 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1898 else:
1899 # The signature of waitpid() means this shouldn't happen
1900 raise AssertionError()
1901
1902 # All child processes are supposed to be in this dict. But this may
1903 # legitimately happen if a grandchild outlives the child (its parent).
1904 # Then it is reparented under this process, so we might receive
1905 # notification of its exit, even though we didn't start it. We can't have
1906 # any knowledge of such processes, so print a warning.
1907 if pid not in self.job_list.child_procs:
1908 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1909 return W1_OK
1910
1911 proc = self.job_list.child_procs[pid]
1912 if 0:
1913 self.job_list.DebugPrint()
1914
1915 if WIFSIGNALED(status):
1916 term_sig = WTERMSIG(status)
1917 status = 128 + term_sig
1918
1919 # Print newline after Ctrl-C.
1920 if term_sig == SIGINT:
1921 print('')
1922
1923 proc.WhenDone(pid, status)
1924
1925 elif WIFEXITED(status):
1926 status = WEXITSTATUS(status)
1927 #log('exit status: %s', status)
1928 proc.WhenDone(pid, status)
1929
1930 elif WIFSTOPPED(status):
1931 #status = WEXITSTATUS(status)
1932 stop_sig = WSTOPSIG(status)
1933
1934 print_stderr('')
1935 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1936 proc.WhenStopped(stop_sig)
1937
1938 else:
1939 raise AssertionError(status)
1940
1941 self.last_status = status # for wait -n
1942 self.tracer.OnProcessEnd(pid, status)
1943 return W1_OK
1944
1945 def PollNotifications(self):
1946 # type: () -> None
1947 """
1948 Process all pending state changes.
1949 """
1950 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1951 continue