OILS / core / process.py View on Github | oilshell.org

1958 lines, 948 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC, EEXIST
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from display import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_EXCL,
58 O_NONBLOCK,
59 O_NOCTTY,
60 O_RDONLY,
61 O_RDWR,
62 O_WRONLY,
63 O_TRUNC,
64)
65
66from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
67
68if TYPE_CHECKING:
69 from _devbuild.gen.runtime_asdl import cmd_value
70 from _devbuild.gen.syntax_asdl import command_t
71 from builtin import trap_osh
72 from core import optview
73 from core.util import _DebugFile
74 from osh.cmd_eval import CommandEvaluator
75
76NO_FD = -1
77
78# Minimum file descriptor that the shell can use. Other descriptors can be
79# directly used by user programs, e.g. exec 9>&1
80#
81# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
82# This is a compromise between bash (unlimited, but requires crazy
83# bookkeeping), and dash/zsh (10) and mksh (24)
84_SHELL_MIN_FD = 100
85
86# Style for 'jobs' builtin
87STYLE_DEFAULT = 0
88STYLE_LONG = 1
89STYLE_PID_ONLY = 2
90
91# To save on allocations in JobList::GetJobWithSpec()
92CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
93
94
95class ctx_FileCloser(object):
96
97 def __init__(self, f):
98 # type: (mylib.LineReader) -> None
99 self.f = f
100
101 def __enter__(self):
102 # type: () -> None
103 pass
104
105 def __exit__(self, type, value, traceback):
106 # type: (Any, Any, Any) -> None
107 self.f.close()
108
109
110def InitInteractiveShell():
111 # type: () -> None
112 """Called when initializing an interactive shell."""
113
114 # The shell itself should ignore Ctrl-\.
115 pyos.Sigaction(SIGQUIT, SIG_IGN)
116
117 # This prevents Ctrl-Z from suspending OSH in interactive mode.
118 pyos.Sigaction(SIGTSTP, SIG_IGN)
119
120 # More signals from
121 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
122 # (but not SIGCHLD)
123 pyos.Sigaction(SIGTTOU, SIG_IGN)
124 pyos.Sigaction(SIGTTIN, SIG_IGN)
125
126 # Register a callback to receive terminal width changes.
127 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
128
129 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
130 # read() have to handle it
131 pyos.RegisterSignalInterest(SIGWINCH)
132
133
134def SaveFd(fd):
135 # type: (int) -> int
136 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
137 return saved
138
139
140class _RedirFrame(object):
141
142 def __init__(self, saved_fd, orig_fd, forget):
143 # type: (int, int, bool) -> None
144 self.saved_fd = saved_fd
145 self.orig_fd = orig_fd
146 self.forget = forget
147
148
149class _FdFrame(object):
150
151 def __init__(self):
152 # type: () -> None
153 self.saved = [] # type: List[_RedirFrame]
154 self.need_wait = [] # type: List[Process]
155
156 def Forget(self):
157 # type: () -> None
158 """For exec 1>&2."""
159 for rf in reversed(self.saved):
160 if rf.saved_fd != NO_FD and rf.forget:
161 posix.close(rf.saved_fd)
162
163 del self.saved[:] # like list.clear() in Python 3.3
164 del self.need_wait[:]
165
166 def __repr__(self):
167 # type: () -> str
168 return '<_FdFrame %s>' % self.saved
169
170
171class FdState(object):
172 """File descriptor state for the current process.
173
174 For example, you can do 'myfunc > out.txt' without forking. Child
175 processes inherit our state.
176 """
177
178 def __init__(
179 self,
180 errfmt, # type: ui.ErrorFormatter
181 job_control, # type: JobControl
182 job_list, # type: JobList
183 mem, # type: state.Mem
184 tracer, # type: Optional[dev.Tracer]
185 waiter, # type: Optional[Waiter]
186 exec_opts, # type: optview.Exec
187 ):
188 # type: (...) -> None
189 """
190 Args:
191 errfmt: for errors
192 job_list: For keeping track of _HereDocWriterThunk
193 """
194 self.errfmt = errfmt
195 self.job_control = job_control
196 self.job_list = job_list
197 self.cur_frame = _FdFrame() # for the top level
198 self.stack = [self.cur_frame]
199 self.mem = mem
200 self.tracer = tracer
201 self.waiter = waiter
202 self.exec_opts = exec_opts
203
204 def Open(self, path):
205 # type: (str) -> mylib.LineReader
206 """Opens a path for read, but moves it out of the reserved 3-9 fd
207 range.
208
209 Returns:
210 A Python file object. The caller is responsible for Close().
211
212 Raises:
213 IOError or OSError if the path can't be found. (This is Python-induced wart)
214 """
215 fd_mode = O_RDONLY
216 f = self._Open(path, 'r', fd_mode)
217
218 # Hacky downcast
219 return cast('mylib.LineReader', f)
220
221 # used for util.DebugFile
222 def OpenForWrite(self, path):
223 # type: (str) -> mylib.Writer
224 fd_mode = O_CREAT | O_RDWR
225 f = self._Open(path, 'w', fd_mode)
226
227 # Hacky downcast
228 return cast('mylib.Writer', f)
229
230 def _Open(self, path, c_mode, fd_mode):
231 # type: (str, str, int) -> IO[str]
232 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
233
234 # Immediately move it to a new location
235 new_fd = SaveFd(fd)
236 posix.close(fd)
237
238 # Return a Python file handle
239 f = posix.fdopen(new_fd, c_mode) # may raise IOError
240 return f
241
242 def _WriteFdToMem(self, fd_name, fd):
243 # type: (str, int) -> None
244 if self.mem:
245 # setvar, not setref
246 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
247 value.Str(str(fd)))
248
249 def _ReadFdFromMem(self, fd_name):
250 # type: (str) -> int
251 val = self.mem.GetValue(fd_name)
252 if val.tag() == value_e.Str:
253 try:
254 return int(cast(value.Str, val).s)
255 except ValueError:
256 return NO_FD
257 return NO_FD
258
259 def _PushSave(self, fd):
260 # type: (int) -> bool
261 """Save fd to a new location and remember to restore it later."""
262 #log('---- _PushSave %s', fd)
263 ok = True
264 try:
265 new_fd = SaveFd(fd)
266 except (IOError, OSError) as e:
267 ok = False
268 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
269 # open.
270 # This seems to be ignored in dash too in savefd()?
271 if e.errno != EBADF:
272 raise
273 if ok:
274 posix.close(fd)
275 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
276 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
277 else:
278 # if we got EBADF, we still need to close the original on Pop()
279 self._PushClose(fd)
280
281 return ok
282
283 def _PushDup(self, fd1, blame_loc):
284 # type: (int, redir_loc_t) -> int
285 """Save fd2 in a higher range, and dup fd1 onto fd2.
286
287 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
288 """
289 UP_loc = blame_loc
290 if blame_loc.tag() == redir_loc_e.VarName:
291 fd2_name = cast(redir_loc.VarName, UP_loc).name
292 try:
293 # F_DUPFD: GREATER than range
294 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
295 except (IOError, OSError) as e:
296 if e.errno == EBADF:
297 print_stderr('F_DUPFD fd %d: %s' %
298 (fd1, pyutil.strerror(e)))
299 return NO_FD
300 else:
301 raise # this redirect failed
302
303 self._WriteFdToMem(fd2_name, new_fd)
304
305 elif blame_loc.tag() == redir_loc_e.Fd:
306 fd2 = cast(redir_loc.Fd, UP_loc).fd
307
308 if fd1 == fd2:
309 # The user could have asked for it to be open on descriptor 3, but open()
310 # already returned 3, e.g. echo 3>out.txt
311 return NO_FD
312
313 # Check the validity of fd1 before _PushSave(fd2)
314 try:
315 fcntl_.fcntl(fd1, F_GETFD)
316 except (IOError, OSError) as e:
317 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
318 raise
319
320 need_restore = self._PushSave(fd2)
321
322 #log('==== dup2 %s %s\n' % (fd1, fd2))
323 try:
324 posix.dup2(fd1, fd2)
325 except (IOError, OSError) as e:
326 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
327 print_stderr('dup2(%d, %d): %s' %
328 (fd1, fd2, pyutil.strerror(e)))
329
330 # Restore and return error
331 if need_restore:
332 rf = self.cur_frame.saved.pop()
333 posix.dup2(rf.saved_fd, rf.orig_fd)
334 posix.close(rf.saved_fd)
335
336 raise # this redirect failed
337
338 new_fd = fd2
339
340 else:
341 raise AssertionError()
342
343 return new_fd
344
345 def _PushCloseFd(self, blame_loc):
346 # type: (redir_loc_t) -> bool
347 """For 2>&-"""
348 # exec {fd}>&- means close the named descriptor
349
350 UP_loc = blame_loc
351 if blame_loc.tag() == redir_loc_e.VarName:
352 fd_name = cast(redir_loc.VarName, UP_loc).name
353 fd = self._ReadFdFromMem(fd_name)
354 if fd == NO_FD:
355 return False
356
357 elif blame_loc.tag() == redir_loc_e.Fd:
358 fd = cast(redir_loc.Fd, UP_loc).fd
359
360 else:
361 raise AssertionError()
362
363 self._PushSave(fd)
364
365 return True
366
367 def _PushClose(self, fd):
368 # type: (int) -> None
369 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
370
371 def _PushWait(self, proc):
372 # type: (Process) -> None
373 self.cur_frame.need_wait.append(proc)
374
375 def _ApplyRedirect(self, r):
376 # type: (RedirValue) -> None
377 arg = r.arg
378 UP_arg = arg
379 with tagswitch(arg) as case:
380
381 if case(redirect_arg_e.Path):
382 arg = cast(redirect_arg.Path, UP_arg)
383 # noclobber flag is OR'd with other flags when allowed
384 noclobber_mode = O_EXCL if self.exec_opts.noclobber() else 0
385 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
386 # NOTE: This is different than >| because it respects noclobber, but
387 # that option is almost never used. See test/wild.sh.
388 mode = O_CREAT | O_WRONLY | O_TRUNC | noclobber_mode
389 elif r.op_id == Id.Redir_Clobber: # >|
390 mode = O_CREAT | O_WRONLY | O_TRUNC
391 elif r.op_id in (Id.Redir_DGreat,
392 Id.Redir_AndDGreat): # >> &>>
393 mode = O_CREAT | O_WRONLY | O_APPEND | noclobber_mode
394 elif r.op_id == Id.Redir_Less: # <
395 mode = O_RDONLY
396 elif r.op_id == Id.Redir_LessGreat: # <>
397 mode = O_CREAT | O_RDWR
398 else:
399 raise NotImplementedError(r.op_id)
400
401 # NOTE: 0666 is affected by umask, all shells use it.
402 try:
403 open_fd = posix.open(arg.filename, mode, 0o666)
404 except (IOError, OSError) as e:
405 if e.errno == EEXIST and self.exec_opts.noclobber():
406 extra = ' (noclobber)'
407 else:
408 extra = ''
409 self.errfmt.Print_(
410 "Can't open %r: %s%s" %
411 (arg.filename, pyutil.strerror(e), extra),
412 blame_loc=r.op_loc)
413 raise # redirect failed
414
415 new_fd = self._PushDup(open_fd, r.loc)
416 if new_fd != NO_FD:
417 posix.close(open_fd)
418
419 # Now handle &> and &>> and their variants. These pairs are the same:
420 #
421 # stdout_stderr.py &> out-err.txt
422 # stdout_stderr.py > out-err.txt 2>&1
423 #
424 # stdout_stderr.py 3&> out-err.txt
425 # stdout_stderr.py 3> out-err.txt 2>&3
426 #
427 # Ditto for {fd}> and {fd}&>
428
429 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
430 self._PushDup(new_fd, redir_loc.Fd(2))
431
432 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
433 arg = cast(redirect_arg.CopyFd, UP_arg)
434
435 if r.op_id == Id.Redir_GreatAnd: # 1>&2
436 self._PushDup(arg.target_fd, r.loc)
437
438 elif r.op_id == Id.Redir_LessAnd: # 0<&5
439 # The only difference between >& and <& is the default file
440 # descriptor argument.
441 self._PushDup(arg.target_fd, r.loc)
442
443 else:
444 raise NotImplementedError()
445
446 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
447 arg = cast(redirect_arg.MoveFd, UP_arg)
448 new_fd = self._PushDup(arg.target_fd, r.loc)
449 if new_fd != NO_FD:
450 posix.close(arg.target_fd)
451
452 UP_loc = r.loc
453 if r.loc.tag() == redir_loc_e.Fd:
454 fd = cast(redir_loc.Fd, UP_loc).fd
455 else:
456 fd = NO_FD
457
458 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
459
460 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
461 self._PushCloseFd(r.loc)
462
463 elif case(redirect_arg_e.HereDoc):
464 arg = cast(redirect_arg.HereDoc, UP_arg)
465
466 # NOTE: Do these descriptors have to be moved out of the range 0-9?
467 read_fd, write_fd = posix.pipe()
468
469 self._PushDup(read_fd, r.loc) # stdin is now the pipe
470
471 # We can't close like we do in the filename case above? The writer can
472 # get a "broken pipe".
473 self._PushClose(read_fd)
474
475 thunk = _HereDocWriterThunk(write_fd, arg.body)
476
477 # Use PIPE_SIZE to save a process in the case of small here
478 # docs, which are the common case. (dash does this.)
479
480 # Note: could instrument this to see how often it happens.
481 # Though strace -ff can also work.
482 start_process = len(arg.body) > 4096
483 #start_process = True
484
485 if start_process:
486 here_proc = Process(thunk, self.job_control, self.job_list,
487 self.tracer)
488
489 # NOTE: we could close the read pipe here, but it doesn't really
490 # matter because we control the code.
491 here_proc.StartProcess(trace.HereDoc)
492 #log('Started %s as %d', here_proc, pid)
493 self._PushWait(here_proc)
494
495 # Now that we've started the child, close it in the parent.
496 posix.close(write_fd)
497
498 else:
499 posix.write(write_fd, arg.body)
500 posix.close(write_fd)
501
502 def Push(self, redirects, err_out):
503 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
504 """Apply a group of redirects and remember to undo them."""
505
506 #log('> fd_state.Push %s', redirects)
507 new_frame = _FdFrame()
508 self.stack.append(new_frame)
509 self.cur_frame = new_frame
510
511 for r in redirects:
512 #log('apply %s', r)
513 with ui.ctx_Location(self.errfmt, r.op_loc):
514 try:
515 self._ApplyRedirect(r)
516 except (IOError, OSError) as e:
517 err_out.append(e)
518 # This can fail too
519 self.Pop(err_out)
520 return # for bad descriptor, etc.
521
522 def PushStdinFromPipe(self, r):
523 # type: (int) -> bool
524 """Save the current stdin and make it come from descriptor 'r'.
525
526 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
527 semantics of
528
529 echo foo | read line; echo $line
530 """
531 new_frame = _FdFrame()
532 self.stack.append(new_frame)
533 self.cur_frame = new_frame
534
535 self._PushDup(r, redir_loc.Fd(0))
536 return True
537
538 def Pop(self, err_out):
539 # type: (List[error.IOError_OSError]) -> None
540 frame = self.stack.pop()
541 #log('< Pop %s', frame)
542 for rf in reversed(frame.saved):
543 if rf.saved_fd == NO_FD:
544 #log('Close %d', orig)
545 try:
546 posix.close(rf.orig_fd)
547 except (IOError, OSError) as e:
548 err_out.append(e)
549 log('Error closing descriptor %d: %s', rf.orig_fd,
550 pyutil.strerror(e))
551 return
552 else:
553 try:
554 posix.dup2(rf.saved_fd, rf.orig_fd)
555 except (IOError, OSError) as e:
556 err_out.append(e)
557 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
558 pyutil.strerror(e))
559 #log('fd state:')
560 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
561 return
562 posix.close(rf.saved_fd)
563 #log('dup2 %s %s', saved, orig)
564
565 # Wait for here doc processes to finish.
566 for proc in frame.need_wait:
567 unused_status = proc.Wait(self.waiter)
568
569 def MakePermanent(self):
570 # type: () -> None
571 self.cur_frame.Forget()
572
573
574class ChildStateChange(object):
575
576 def __init__(self):
577 # type: () -> None
578 """Empty constructor for mycpp."""
579 pass
580
581 def Apply(self):
582 # type: () -> None
583 raise NotImplementedError()
584
585 def ApplyFromParent(self, proc):
586 # type: (Process) -> None
587 """Noop for all state changes other than SetPgid for mycpp."""
588 pass
589
590
591class StdinFromPipe(ChildStateChange):
592
593 def __init__(self, pipe_read_fd, w):
594 # type: (int, int) -> None
595 self.r = pipe_read_fd
596 self.w = w
597
598 def __repr__(self):
599 # type: () -> str
600 return '<StdinFromPipe %d %d>' % (self.r, self.w)
601
602 def Apply(self):
603 # type: () -> None
604 posix.dup2(self.r, 0)
605 posix.close(self.r) # close after dup
606
607 posix.close(self.w) # we're reading from the pipe, not writing
608 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
609
610
611class StdoutToPipe(ChildStateChange):
612
613 def __init__(self, r, pipe_write_fd):
614 # type: (int, int) -> None
615 self.r = r
616 self.w = pipe_write_fd
617
618 def __repr__(self):
619 # type: () -> str
620 return '<StdoutToPipe %d %d>' % (self.r, self.w)
621
622 def Apply(self):
623 # type: () -> None
624 posix.dup2(self.w, 1)
625 posix.close(self.w) # close after dup
626
627 posix.close(self.r) # we're writing to the pipe, not reading
628 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
629
630
631INVALID_PGID = -1
632# argument to setpgid() that means the process is its own leader
633OWN_LEADER = 0
634
635
636class SetPgid(ChildStateChange):
637
638 def __init__(self, pgid, tracer):
639 # type: (int, dev.Tracer) -> None
640 self.pgid = pgid
641 self.tracer = tracer
642
643 def Apply(self):
644 # type: () -> None
645 try:
646 posix.setpgid(0, self.pgid)
647 except (IOError, OSError) as e:
648 self.tracer.OtherMessage(
649 'osh: child %d failed to set its process group to %d: %s' %
650 (posix.getpid(), self.pgid, pyutil.strerror(e)))
651
652 def ApplyFromParent(self, proc):
653 # type: (Process) -> None
654 try:
655 posix.setpgid(proc.pid, self.pgid)
656 except (IOError, OSError) as e:
657 self.tracer.OtherMessage(
658 'osh: parent failed to set process group for PID %d to %d: %s'
659 % (proc.pid, self.pgid, pyutil.strerror(e)))
660
661
662class ExternalProgram(object):
663 """The capability to execute an external program like 'ls'."""
664
665 def __init__(
666 self,
667 hijack_shebang, # type: str
668 fd_state, # type: FdState
669 errfmt, # type: ui.ErrorFormatter
670 debug_f, # type: _DebugFile
671 ):
672 # type: (...) -> None
673 """
674 Args:
675 hijack_shebang: The path of an interpreter to run instead of the one
676 specified in the shebang line. May be empty.
677 """
678 self.hijack_shebang = hijack_shebang
679 self.fd_state = fd_state
680 self.errfmt = errfmt
681 self.debug_f = debug_f
682
683 def Exec(self, argv0_path, cmd_val, environ):
684 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
685 """Execute a program and exit this process.
686
687 Called by: ls / exec ls / ( ls / )
688 """
689 probe('process', 'ExternalProgram_Exec', argv0_path)
690 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
691 True)
692 assert False, "This line should never execute" # NO RETURN
693
694 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
695 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
696 if len(self.hijack_shebang):
697 opened = True
698 try:
699 f = self.fd_state.Open(argv0_path)
700 except (IOError, OSError) as e:
701 opened = False
702
703 if opened:
704 with ctx_FileCloser(f):
705 # Test if the shebang looks like a shell. TODO: The file might be
706 # binary with no newlines, so read 80 bytes instead of readline().
707
708 #line = f.read(80) # type: ignore # TODO: fix this
709 line = f.readline()
710
711 if match.ShouldHijack(line):
712 h_argv = [self.hijack_shebang, argv0_path]
713 h_argv.extend(argv[1:])
714 argv = h_argv
715 argv0_path = self.hijack_shebang
716 self.debug_f.writeln('Hijacked: %s' % argv0_path)
717 else:
718 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
719 pass
720
721 try:
722 posix.execve(argv0_path, argv, environ)
723 except (IOError, OSError) as e:
724 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
725 if e.errno == ENOEXEC and should_retry:
726 new_argv = ['/bin/sh', argv0_path]
727 new_argv.extend(argv[1:])
728 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
729 # NO RETURN
730
731 # Would be nice: when the path is relative and ENOENT: print PWD and do
732 # spelling correction?
733
734 self.errfmt.Print_(
735 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
736 argv0_loc)
737
738 # POSIX mentions 126 and 127 for two specific errors. The rest are
739 # unspecified.
740 #
741 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
742 if e.errno == EACCES:
743 status = 126
744 elif e.errno == ENOENT:
745 # TODO: most shells print 'command not found', rather than strerror()
746 # == "No such file or directory". That's better because it's at the
747 # end of the path search, and we're never searching for a directory.
748 status = 127
749 else:
750 # dash uses 2, but we use that for parse errors. This seems to be
751 # consistent with mksh and zsh.
752 status = 127
753
754 posix._exit(status)
755 # NO RETURN
756
757
758class Thunk(object):
759 """Abstract base class for things runnable in another process."""
760
761 def __init__(self):
762 # type: () -> None
763 """Empty constructor for mycpp."""
764 pass
765
766 def Run(self):
767 # type: () -> None
768 """Returns a status code."""
769 raise NotImplementedError()
770
771 def UserString(self):
772 # type: () -> str
773 """Display for the 'jobs' list."""
774 raise NotImplementedError()
775
776 def __repr__(self):
777 # type: () -> str
778 return self.UserString()
779
780
781class ExternalThunk(Thunk):
782 """An external executable."""
783
784 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
785 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
786 self.ext_prog = ext_prog
787 self.argv0_path = argv0_path
788 self.cmd_val = cmd_val
789 self.environ = environ
790
791 def UserString(self):
792 # type: () -> str
793
794 # NOTE: This is the format the Tracer uses.
795 # bash displays sleep $n & (code)
796 # but OSH displays sleep 1 & (argv array)
797 # We could switch the former but I'm not sure it's necessary.
798 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
799 return '[process] %s' % ' '.join(tmp)
800
801 def Run(self):
802 # type: () -> None
803 """An ExternalThunk is run in parent for the exec builtin."""
804 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
805
806
807class SubProgramThunk(Thunk):
808 """A subprogram that can be executed in another process."""
809
810 def __init__(self, cmd_ev, node, trap_state, multi_trace, inherit_errexit,
811 inherit_errtrace):
812 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool, bool) -> None
813 self.cmd_ev = cmd_ev
814 self.node = node
815 self.trap_state = trap_state
816 self.multi_trace = multi_trace
817 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
818 self.inherit_errtrace = inherit_errtrace # for bash errtrace compatibility
819
820 def UserString(self):
821 # type: () -> str
822
823 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
824 # TODO: Extract SPIDS from node to display source? Note that
825 # CompoundStatus also has locations of each pipeline component; see
826 # Executor.RunPipeline()
827 thunk_str = ui.CommandType(self.node)
828 return '[subprog] %s' % thunk_str
829
830 def Run(self):
831 # type: () -> None
832 #self.errfmt.OneLineErrExit() # don't quote code in child processes
833 probe('process', 'SubProgramThunk_Run')
834
835 # TODO: break circular dep. Bit flags could go in ASDL or headers.
836 from osh import cmd_eval
837
838 # signal handlers aren't inherited
839 self.trap_state.ClearForSubProgram(self.inherit_errtrace)
840
841 # NOTE: may NOT return due to exec().
842 if not self.inherit_errexit:
843 self.cmd_ev.mutable_opts.DisableErrExit()
844 try:
845 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
846 self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
847 status = self.cmd_ev.LastStatus()
848 # NOTE: We ignore the is_fatal return value. The user should set -o
849 # errexit so failures in subprocesses cause failures in the parent.
850 except util.UserExit as e:
851 status = e.status
852
853 # Handle errors in a subshell. These two cases are repeated from main()
854 # and the core/completion.py hook.
855 except KeyboardInterrupt:
856 print('')
857 status = 130 # 128 + 2
858 except (IOError, OSError) as e:
859 print_stderr('oils I/O error (subprogram): %s' %
860 pyutil.strerror(e))
861 status = 2
862
863 # If ProcessInit() doesn't turn off buffering, this is needed before
864 # _exit()
865 pyos.FlushStdout()
866
867 self.multi_trace.WriteDumps()
868
869 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
870 # gets called in BOTH processes.
871 # The crash dump seems to be unaffected.
872 posix._exit(status)
873
874
875class _HereDocWriterThunk(Thunk):
876 """Write a here doc to one end of a pipe.
877
878 May be be executed in either a child process or the main shell
879 process.
880 """
881
882 def __init__(self, w, body_str):
883 # type: (int, str) -> None
884 self.w = w
885 self.body_str = body_str
886
887 def UserString(self):
888 # type: () -> str
889
890 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
891 # shells don't have this problem because they use temp files! That's a bit
892 # unfortunate.
893 return '[here doc writer]'
894
895 def Run(self):
896 # type: () -> None
897 """do_exit: For small pipelines."""
898 probe('process', 'HereDocWriterThunk_Run')
899 #log('Writing %r', self.body_str)
900 posix.write(self.w, self.body_str)
901 #log('Wrote %r', self.body_str)
902 posix.close(self.w)
903 #log('Closed %d', self.w)
904
905 posix._exit(0)
906
907
908class Job(object):
909 """Interface for both Process and Pipeline.
910
911 They both can be put in the background and waited on.
912
913 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
914
915 sleep 1 | sleep 2 &
916
917 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
918 not a JOB ID.
919 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
920 - The process group leader (setpgid) is the FIRST PID.
921 - It's also %1 or %+. The last job started.
922 """
923
924 def __init__(self):
925 # type: () -> None
926 # Initial state with & or Ctrl-Z is Running.
927 self.state = job_state_e.Running
928 self.job_id = -1
929 self.in_background = False
930
931 def DisplayJob(self, job_id, f, style):
932 # type: (int, mylib.Writer, int) -> None
933 raise NotImplementedError()
934
935 def State(self):
936 # type: () -> job_state_t
937 return self.state
938
939 def ProcessGroupId(self):
940 # type: () -> int
941 """Return the process group ID associated with this job."""
942 raise NotImplementedError()
943
944 def JobWait(self, waiter):
945 # type: (Waiter) -> wait_status_t
946 """Wait for this process/pipeline to be stopped or finished."""
947 raise NotImplementedError()
948
949 def SetBackground(self):
950 # type: () -> None
951 """Record that this job is running in the background."""
952 self.in_background = True
953
954 def SetForeground(self):
955 # type: () -> None
956 """Record that this job is running in the foreground."""
957 self.in_background = False
958
959
960class Process(Job):
961 """A process to run.
962
963 TODO: Should we make it clear that this is a FOREGROUND process? A
964 background process is wrapped in a "job". It is unevaluated.
965
966 It provides an API to manipulate file descriptor state in parent and child.
967 """
968
969 def __init__(self, thunk, job_control, job_list, tracer):
970 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
971 """
972 Args:
973 thunk: Thunk instance
974 job_list: for process bookkeeping
975 """
976 Job.__init__(self)
977 assert isinstance(thunk, Thunk), thunk
978 self.thunk = thunk
979 self.job_control = job_control
980 self.job_list = job_list
981 self.tracer = tracer
982
983 # For pipelines
984 self.parent_pipeline = None # type: Pipeline
985 self.state_changes = [] # type: List[ChildStateChange]
986 self.close_r = -1
987 self.close_w = -1
988
989 self.pid = -1
990 self.status = -1
991
992 def Init_ParentPipeline(self, pi):
993 # type: (Pipeline) -> None
994 """For updating PIPESTATUS."""
995 self.parent_pipeline = pi
996
997 def __repr__(self):
998 # type: () -> str
999
1000 # note: be wary of infinite mutual recursion
1001 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
1002 #return '<Process %s%s>' % (self.thunk, s)
1003 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
1004
1005 def ProcessGroupId(self):
1006 # type: () -> int
1007 """Returns the group ID of this process."""
1008 # This should only ever be called AFTER the process has started
1009 assert self.pid != -1
1010 if self.parent_pipeline:
1011 # XXX: Maybe we should die here instead? Unclear if this branch
1012 # should even be reachable with the current builtins.
1013 return self.parent_pipeline.ProcessGroupId()
1014
1015 return self.pid
1016
1017 def DisplayJob(self, job_id, f, style):
1018 # type: (int, mylib.Writer, int) -> None
1019 if job_id == -1:
1020 job_id_str = ' '
1021 else:
1022 job_id_str = '%%%d' % job_id
1023 if style == STYLE_PID_ONLY:
1024 f.write('%d\n' % self.pid)
1025 else:
1026 f.write('%s %d %7s ' %
1027 (job_id_str, self.pid, _JobStateStr(self.state)))
1028 f.write(self.thunk.UserString())
1029 f.write('\n')
1030
1031 def AddStateChange(self, s):
1032 # type: (ChildStateChange) -> None
1033 self.state_changes.append(s)
1034
1035 def AddPipeToClose(self, r, w):
1036 # type: (int, int) -> None
1037 self.close_r = r
1038 self.close_w = w
1039
1040 def MaybeClosePipe(self):
1041 # type: () -> None
1042 if self.close_r != -1:
1043 posix.close(self.close_r)
1044 posix.close(self.close_w)
1045
1046 def StartProcess(self, why):
1047 # type: (trace_t) -> int
1048 """Start this process with fork(), handling redirects."""
1049 pid = posix.fork()
1050 if pid < 0:
1051 # When does this happen?
1052 e_die('Fatal error in posix.fork()')
1053
1054 elif pid == 0: # child
1055 # Note: this happens in BOTH interactive and non-interactive shells.
1056 # We technically don't need to do most of it in non-interactive, since we
1057 # did not change state in InitInteractiveShell().
1058
1059 for st in self.state_changes:
1060 st.Apply()
1061
1062 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1063 # shouldn't have this.
1064 # https://docs.python.org/2/library/signal.html
1065 # See Python/pythonrun.c.
1066 pyos.Sigaction(SIGPIPE, SIG_DFL)
1067
1068 # Respond to Ctrl-\ (core dump)
1069 pyos.Sigaction(SIGQUIT, SIG_DFL)
1070
1071 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1072 # foreground because suspending them is difficult with our 'lastpipe'
1073 # semantics.
1074 pid = posix.getpid()
1075 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1076 pyos.Sigaction(SIGTSTP, SIG_DFL)
1077
1078 # More signals from
1079 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1080 # (but not SIGCHLD)
1081 pyos.Sigaction(SIGTTOU, SIG_DFL)
1082 pyos.Sigaction(SIGTTIN, SIG_DFL)
1083
1084 self.tracer.OnNewProcess(pid)
1085 # clear foreground pipeline for subshells
1086 self.thunk.Run()
1087 # Never returns
1088
1089 #log('STARTED process %s, pid = %d', self, pid)
1090 self.tracer.OnProcessStart(pid, why)
1091
1092 # Class invariant: after the process is started, it stores its PID.
1093 self.pid = pid
1094
1095 # SetPgid needs to be applied from the child and the parent to avoid
1096 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1097 for st in self.state_changes:
1098 st.ApplyFromParent(self)
1099
1100 # Program invariant: We keep track of every child process!
1101 self.job_list.AddChildProcess(pid, self)
1102
1103 return pid
1104
1105 def Wait(self, waiter):
1106 # type: (Waiter) -> int
1107 """Wait for this process to finish."""
1108 while self.state == job_state_e.Running:
1109 # Only return if there's nothing to wait for. Keep waiting if we were
1110 # interrupted with a signal.
1111 if waiter.WaitForOne() == W1_ECHILD:
1112 break
1113
1114 assert self.status >= 0, self.status
1115 return self.status
1116
1117 def JobWait(self, waiter):
1118 # type: (Waiter) -> wait_status_t
1119 # wait builtin can be interrupted
1120 while self.state == job_state_e.Running:
1121 result = waiter.WaitForOne()
1122
1123 if result >= 0: # signal
1124 return wait_status.Cancelled(result)
1125
1126 if result == W1_ECHILD:
1127 break
1128
1129 return wait_status.Proc(self.status)
1130
1131 def WhenStopped(self, stop_sig):
1132 # type: (int) -> None
1133
1134 # 128 is a shell thing
1135 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1136 self.status = 128 + stop_sig
1137 self.state = job_state_e.Stopped
1138
1139 if self.job_id == -1:
1140 # This process was started in the foreground
1141 self.job_list.AddJob(self)
1142
1143 if not self.in_background:
1144 self.job_control.MaybeTakeTerminal()
1145 self.SetBackground()
1146
1147 def WhenDone(self, pid, status):
1148 # type: (int, int) -> None
1149 """Called by the Waiter when this Process finishes."""
1150
1151 #log('Process WhenDone %d %d', pid, status)
1152 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1153 self.status = status
1154 self.state = job_state_e.Done
1155 if self.parent_pipeline:
1156 self.parent_pipeline.WhenDone(pid, status)
1157 else:
1158 if self.job_id != -1:
1159 # Job might have been brought to the foreground after being
1160 # assigned a job ID.
1161 if self.in_background:
1162 # TODO: bash only prints this interactively
1163 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1164
1165 self.job_list.RemoveJob(self.job_id)
1166
1167 self.job_list.RemoveChildProcess(self.pid)
1168
1169 if not self.in_background:
1170 self.job_control.MaybeTakeTerminal()
1171
1172 def RunProcess(self, waiter, why):
1173 # type: (Waiter, trace_t) -> int
1174 """Run this process synchronously."""
1175 self.StartProcess(why)
1176 # ShellExecutor might be calling this for the last part of a pipeline.
1177 if self.parent_pipeline is None:
1178 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1179 # calling getpgid()?
1180 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1181 return self.Wait(waiter)
1182
1183
1184class ctx_Pipe(object):
1185
1186 def __init__(self, fd_state, fd, err_out):
1187 # type: (FdState, int, List[error.IOError_OSError]) -> None
1188 fd_state.PushStdinFromPipe(fd)
1189 self.fd_state = fd_state
1190 self.err_out = err_out
1191
1192 def __enter__(self):
1193 # type: () -> None
1194 pass
1195
1196 def __exit__(self, type, value, traceback):
1197 # type: (Any, Any, Any) -> None
1198 self.fd_state.Pop(self.err_out)
1199
1200
1201class Pipeline(Job):
1202 """A pipeline of processes to run.
1203
1204 Cases we handle:
1205
1206 foo | bar
1207 $(foo | bar)
1208 foo | bar | read v
1209 """
1210
1211 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1212 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1213 Job.__init__(self)
1214 self.job_control = job_control
1215 self.job_list = job_list
1216 self.tracer = tracer
1217
1218 self.procs = [] # type: List[Process]
1219 self.pids = [] # type: List[int] # pids in order
1220 self.pipe_status = [] # type: List[int] # status in order
1221 self.status = -1 # for 'wait' jobs
1222
1223 self.pgid = INVALID_PGID
1224
1225 # Optional for foreground
1226 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1227 self.last_pipe = None # type: Tuple[int, int]
1228
1229 self.sigpipe_status_ok = sigpipe_status_ok
1230
1231 def ProcessGroupId(self):
1232 # type: () -> int
1233 """Returns the group ID of this pipeline."""
1234 return self.pgid
1235
1236 def DisplayJob(self, job_id, f, style):
1237 # type: (int, mylib.Writer, int) -> None
1238 if style == STYLE_PID_ONLY:
1239 f.write('%d\n' % self.procs[0].pid)
1240 else:
1241 # Note: this is STYLE_LONG.
1242 for i, proc in enumerate(self.procs):
1243 if i == 0: # show job ID for first element in pipeline
1244 job_id_str = '%%%d' % job_id
1245 else:
1246 job_id_str = ' ' # 2 spaces
1247
1248 f.write('%s %d %7s ' %
1249 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1250 f.write(proc.thunk.UserString())
1251 f.write('\n')
1252
1253 def DebugPrint(self):
1254 # type: () -> None
1255 print('Pipeline in state %s' % _JobStateStr(self.state))
1256 if mylib.PYTHON: # %s for Process not allowed in C++
1257 for proc in self.procs:
1258 print(' proc %s' % proc)
1259 _, last_node = self.last_thunk
1260 print(' last %s' % last_node)
1261 print(' pipe_status %s' % self.pipe_status)
1262
1263 def Add(self, p):
1264 # type: (Process) -> None
1265 """Append a process to the pipeline."""
1266 if len(self.procs) == 0:
1267 self.procs.append(p)
1268 return
1269
1270 r, w = posix.pipe()
1271 #log('pipe for %s: %d %d', p, r, w)
1272 prev = self.procs[-1]
1273
1274 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1275 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1276
1277 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1278
1279 self.procs.append(p)
1280
1281 def AddLast(self, thunk):
1282 # type: (Tuple[CommandEvaluator, command_t]) -> None
1283 """Append the last noden to the pipeline.
1284
1285 This is run in the CURRENT process. It is OPTIONAL, because
1286 pipelines in the background are run uniformly.
1287 """
1288 self.last_thunk = thunk
1289
1290 assert len(self.procs) != 0
1291
1292 r, w = posix.pipe()
1293 prev = self.procs[-1]
1294 prev.AddStateChange(StdoutToPipe(r, w))
1295
1296 self.last_pipe = (r, w) # So we can connect it to last_thunk
1297
1298 def StartPipeline(self, waiter):
1299 # type: (Waiter) -> None
1300
1301 # If we are creating a pipeline in a subshell or we aren't running with job
1302 # control, our children should remain in our inherited process group.
1303 # the pipelines's group ID.
1304 if self.job_control.Enabled():
1305 self.pgid = OWN_LEADER # first process in pipeline is the leader
1306
1307 for i, proc in enumerate(self.procs):
1308 if self.pgid != INVALID_PGID:
1309 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1310
1311 # Figure out the pid
1312 pid = proc.StartProcess(trace.PipelinePart)
1313 if i == 0 and self.pgid != INVALID_PGID:
1314 # Mimic bash and use the PID of the FIRST process as the group for the
1315 # whole pipeline.
1316 self.pgid = pid
1317
1318 self.pids.append(pid)
1319 self.pipe_status.append(-1) # uninitialized
1320
1321 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1322 # It can't be done at the end; otherwise processes will have descriptors
1323 # from non-adjacent pipes.
1324 proc.MaybeClosePipe()
1325
1326 if self.last_thunk:
1327 self.pipe_status.append(-1) # for self.last_thunk
1328
1329 def LastPid(self):
1330 # type: () -> int
1331 """For the odd $! variable.
1332
1333 It would be better if job IDs or PGIDs were used consistently.
1334 """
1335 return self.pids[-1]
1336
1337 def Wait(self, waiter):
1338 # type: (Waiter) -> List[int]
1339 """Wait for this pipeline to finish."""
1340
1341 assert self.procs, "no procs for Wait()"
1342 # waitpid(-1) zero or more times
1343 while self.state == job_state_e.Running:
1344 # Keep waiting until there's nothing to wait for.
1345 if waiter.WaitForOne() == W1_ECHILD:
1346 break
1347
1348 return self.pipe_status
1349
1350 def JobWait(self, waiter):
1351 # type: (Waiter) -> wait_status_t
1352 """Called by 'wait' builtin, e.g. 'wait %1'."""
1353 # wait builtin can be interrupted
1354 assert self.procs, "no procs for Wait()"
1355 while self.state == job_state_e.Running:
1356 result = waiter.WaitForOne()
1357
1358 if result >= 0: # signal
1359 return wait_status.Cancelled(result)
1360
1361 if result == W1_ECHILD:
1362 break
1363
1364 return wait_status.Pipeline(self.pipe_status)
1365
1366 def RunLastPart(self, waiter, fd_state):
1367 # type: (Waiter, FdState) -> List[int]
1368 """Run this pipeline synchronously (foreground pipeline).
1369
1370 Returns:
1371 pipe_status (list of integers).
1372 """
1373 assert len(self.pids) == len(self.procs)
1374
1375 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1376 from osh import cmd_eval
1377
1378 # This is tcsetpgrp()
1379 # TODO: fix race condition -- I believe the first process could have
1380 # stopped already, and thus getpgid() will fail
1381 self.job_control.MaybeGiveTerminal(self.pgid)
1382
1383 # Run the last part of the pipeline IN PARALLEL with other processes. It
1384 # may or may not fork:
1385 # echo foo | read line # no fork, the builtin runs in THIS shell process
1386 # ls | wc -l # fork for 'wc'
1387
1388 cmd_ev, last_node = self.last_thunk
1389
1390 assert self.last_pipe is not None
1391 r, w = self.last_pipe # set in AddLast()
1392 posix.close(w) # we will not write here
1393
1394 # Fix lastpipe / job control / DEBUG trap interaction
1395 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1396
1397 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1398 # a pipeline.
1399 cmd_flags |= cmd_eval.NoErrTrap
1400
1401 io_errors = [] # type: List[error.IOError_OSError]
1402 with ctx_Pipe(fd_state, r, io_errors):
1403 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1404
1405 if len(io_errors):
1406 e_die('Error setting up last part of pipeline: %s' %
1407 pyutil.strerror(io_errors[0]))
1408
1409 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1410 # /dev/urandom | sleep 1' will never get SIGPIPE.
1411 posix.close(r)
1412
1413 self.pipe_status[-1] = cmd_ev.LastStatus()
1414 if self.AllDone():
1415 self.state = job_state_e.Done
1416
1417 #log('pipestatus before all have finished = %s', self.pipe_status)
1418 return self.Wait(waiter)
1419
1420 def AllDone(self):
1421 # type: () -> bool
1422
1423 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1424 for status in self.pipe_status:
1425 if status == -1:
1426 return False
1427 return True
1428
1429 def WhenDone(self, pid, status):
1430 # type: (int, int) -> None
1431 """Called by Process.WhenDone."""
1432 #log('Pipeline WhenDone %d %d', pid, status)
1433 i = self.pids.index(pid)
1434 assert i != -1, 'Unexpected PID %d' % pid
1435
1436 if status == 141 and self.sigpipe_status_ok:
1437 status = 0
1438
1439 self.job_list.RemoveChildProcess(pid)
1440 self.pipe_status[i] = status
1441 if self.AllDone():
1442 if self.job_id != -1:
1443 # Job might have been brought to the foreground after being
1444 # assigned a job ID.
1445 if self.in_background:
1446 print_stderr('[%d] Done PGID %d' %
1447 (self.job_id, self.pids[0]))
1448
1449 self.job_list.RemoveJob(self.job_id)
1450
1451 # status of pipeline is status of last process
1452 self.status = self.pipe_status[-1]
1453 self.state = job_state_e.Done
1454 if not self.in_background:
1455 self.job_control.MaybeTakeTerminal()
1456
1457
1458def _JobStateStr(i):
1459 # type: (job_state_t) -> str
1460 return job_state_str(i)[10:] # remove 'job_state.'
1461
1462
1463def _GetTtyFd():
1464 # type: () -> int
1465 """Returns -1 if stdio is not a TTY."""
1466 try:
1467 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1468 except (IOError, OSError) as e:
1469 return -1
1470
1471
1472class ctx_TerminalControl(object):
1473
1474 def __init__(self, job_control, errfmt):
1475 # type: (JobControl, ui.ErrorFormatter) -> None
1476 job_control.InitJobControl()
1477 self.job_control = job_control
1478 self.errfmt = errfmt
1479
1480 def __enter__(self):
1481 # type: () -> None
1482 pass
1483
1484 def __exit__(self, type, value, traceback):
1485 # type: (Any, Any, Any) -> None
1486
1487 # Return the TTY to the original owner before exiting.
1488 try:
1489 self.job_control.MaybeReturnTerminal()
1490 except error.FatalRuntime as e:
1491 # Don't abort the shell on error, just print a message.
1492 self.errfmt.PrettyPrintError(e)
1493
1494
1495class JobControl(object):
1496 """Interface to setpgid(), tcsetpgrp(), etc."""
1497
1498 def __init__(self):
1499 # type: () -> None
1500
1501 # The main shell's PID and group ID.
1502 self.shell_pid = -1
1503 self.shell_pgid = -1
1504
1505 # The fd of the controlling tty. Set to -1 when job control is disabled.
1506 self.shell_tty_fd = -1
1507
1508 # For giving the terminal back to our parent before exiting (if not a login
1509 # shell).
1510 self.original_tty_pgid = -1
1511
1512 def InitJobControl(self):
1513 # type: () -> None
1514 self.shell_pid = posix.getpid()
1515 orig_shell_pgid = posix.getpgid(0)
1516 self.shell_pgid = orig_shell_pgid
1517 self.shell_tty_fd = _GetTtyFd()
1518
1519 # If we aren't the leader of our process group, create a group and mark
1520 # ourselves as the leader.
1521 if self.shell_pgid != self.shell_pid:
1522 try:
1523 posix.setpgid(self.shell_pid, self.shell_pid)
1524 self.shell_pgid = self.shell_pid
1525 except (IOError, OSError) as e:
1526 self.shell_tty_fd = -1
1527
1528 if self.shell_tty_fd != -1:
1529 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1530
1531 # If stdio is a TTY, put the shell's process group in the foreground.
1532 try:
1533 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1534 except (IOError, OSError) as e:
1535 # We probably aren't in the session leader's process group. Disable job
1536 # control.
1537 self.shell_tty_fd = -1
1538 self.shell_pgid = orig_shell_pgid
1539 posix.setpgid(self.shell_pid, self.shell_pgid)
1540
1541 def Enabled(self):
1542 # type: () -> bool
1543
1544 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1545 # think.
1546 curr_pid = posix.getpid()
1547 # Only the main shell should bother with job control functions.
1548 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1549
1550 # TODO: This isn't a PID. This is a process group ID?
1551 #
1552 # What should the table look like?
1553 #
1554 # Do we need the last PID? I don't know why bash prints that. Probably so
1555 # you can do wait $!
1556 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1557 #
1558 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1559 # job_id is just an integer. This is sort of lame.
1560 #
1561 # [job_id, flag, pgid, job_state, node]
1562
1563 def MaybeGiveTerminal(self, pgid):
1564 # type: (int) -> None
1565 """If stdio is a TTY, move the given process group to the
1566 foreground."""
1567 if not self.Enabled():
1568 # Only call tcsetpgrp when job control is enabled.
1569 return
1570
1571 try:
1572 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1573 except (IOError, OSError) as e:
1574 e_die('osh: Failed to move process group %d to foreground: %s' %
1575 (pgid, pyutil.strerror(e)))
1576
1577 def MaybeTakeTerminal(self):
1578 # type: () -> None
1579 """If stdio is a TTY, return the main shell's process group to the
1580 foreground."""
1581 self.MaybeGiveTerminal(self.shell_pgid)
1582
1583 def MaybeReturnTerminal(self):
1584 # type: () -> None
1585 """Called before the shell exits."""
1586 self.MaybeGiveTerminal(self.original_tty_pgid)
1587
1588
1589class JobList(object):
1590 """Global list of jobs, used by a few builtins."""
1591
1592 def __init__(self):
1593 # type: () -> None
1594
1595 # job_id -> Job instance
1596 self.jobs = {} # type: Dict[int, Job]
1597
1598 # pid -> Process. This is for STOP notification.
1599 self.child_procs = {} # type: Dict[int, Process]
1600 self.debug_pipelines = [] # type: List[Pipeline]
1601
1602 # Counter used to assign IDs to jobs. It is incremented every time a job
1603 # is created. Once all active jobs are done it is reset to 1. I'm not
1604 # sure if this reset behavior is mandated by POSIX, but other shells do
1605 # it, so we mimic for the sake of compatibility.
1606 self.job_id = 1
1607
1608 def AddJob(self, job):
1609 # type: (Job) -> int
1610 """Add a background job to the list.
1611
1612 A job is either a Process or Pipeline. You can resume a job with 'fg',
1613 kill it with 'kill', etc.
1614
1615 Two cases:
1616
1617 1. async jobs: sleep 5 | sleep 4 &
1618 2. stopped jobs: sleep 5; then Ctrl-Z
1619 """
1620 job_id = self.job_id
1621 self.jobs[job_id] = job
1622 job.job_id = job_id
1623 self.job_id += 1
1624 return job_id
1625
1626 def RemoveJob(self, job_id):
1627 # type: (int) -> None
1628 """Process and Pipeline can call this."""
1629 mylib.dict_erase(self.jobs, job_id)
1630
1631 if len(self.jobs) == 0:
1632 self.job_id = 1
1633
1634 def AddChildProcess(self, pid, proc):
1635 # type: (int, Process) -> None
1636 """Every child process should be added here as soon as we know its PID.
1637
1638 When the Waiter gets an EXITED or STOPPED notification, we need
1639 to know about it so 'jobs' can work.
1640 """
1641 self.child_procs[pid] = proc
1642
1643 def RemoveChildProcess(self, pid):
1644 # type: (int) -> None
1645 """Remove the child process with the given PID."""
1646 mylib.dict_erase(self.child_procs, pid)
1647
1648 if mylib.PYTHON:
1649
1650 def AddPipeline(self, pi):
1651 # type: (Pipeline) -> None
1652 """For debugging only."""
1653 self.debug_pipelines.append(pi)
1654
1655 def ProcessFromPid(self, pid):
1656 # type: (int) -> Process
1657 """For wait $PID.
1658
1659 There's no way to wait for a pipeline with a PID. That uses job
1660 syntax, e.g. %1. Not a great interface.
1661 """
1662 return self.child_procs.get(pid)
1663
1664 def GetCurrentAndPreviousJobs(self):
1665 # type: () -> Tuple[Optional[Job], Optional[Job]]
1666 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1667
1668 See the POSIX specification for the `jobs` builtin for details:
1669 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1670
1671 IMPORTANT NOTE: This method assumes that the jobs list will not change
1672 during its execution! This assumption holds for now because we only ever
1673 update the jobs list from the main loop after WaitPid() informs us of a
1674 change. If we implement `set -b` and install a signal handler for
1675 SIGCHLD we should be careful to synchronize it with this function. The
1676 unsafety of mutating GC data structures from a signal handler should
1677 make this a non-issue, but if bugs related to this appear this note may
1678 be helpful...
1679 """
1680 # Split all active jobs by state and sort each group by decreasing job
1681 # ID to approximate newness.
1682 stopped_jobs = [] # type: List[Job]
1683 running_jobs = [] # type: List[Job]
1684 for i in xrange(0, self.job_id):
1685 job = self.jobs.get(i, None)
1686 if not job:
1687 continue
1688
1689 if job.state == job_state_e.Stopped:
1690 stopped_jobs.append(job)
1691
1692 elif job.state == job_state_e.Running:
1693 running_jobs.append(job)
1694
1695 current = None # type: Optional[Job]
1696 previous = None # type: Optional[Job]
1697 # POSIX says: If there is any suspended job, then the current job shall
1698 # be a suspended job. If there are at least two suspended jobs, then the
1699 # previous job also shall be a suspended job.
1700 #
1701 # So, we will only return running jobs from here if there are no recent
1702 # stopped jobs.
1703 if len(stopped_jobs) > 0:
1704 current = stopped_jobs.pop()
1705
1706 if len(stopped_jobs) > 0:
1707 previous = stopped_jobs.pop()
1708
1709 if len(running_jobs) > 0 and not current:
1710 current = running_jobs.pop()
1711
1712 if len(running_jobs) > 0 and not previous:
1713 previous = running_jobs.pop()
1714
1715 if not previous:
1716 previous = current
1717
1718 return current, previous
1719
1720 def GetJobWithSpec(self, job_spec):
1721 # type: (str) -> Optional[Job]
1722 """Parse the given job spec and return the matching job. If there is no
1723 matching job, this function returns None.
1724
1725 See the POSIX spec for the `jobs` builtin for details about job specs:
1726 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1727 """
1728 if job_spec in CURRENT_JOB_SPECS:
1729 current, _ = self.GetCurrentAndPreviousJobs()
1730 return current
1731
1732 if job_spec == '%-':
1733 _, previous = self.GetCurrentAndPreviousJobs()
1734 return previous
1735
1736 # TODO: Add support for job specs based on prefixes of process argv.
1737 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1738 if m is not None:
1739 assert len(m) == 2
1740 job_id = int(m[1])
1741 if job_id in self.jobs:
1742 return self.jobs[job_id]
1743
1744 return None
1745
1746 def DisplayJobs(self, style):
1747 # type: (int) -> None
1748 """Used by the 'jobs' builtin.
1749
1750 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1751
1752 "By default, the jobs utility shall display the status of all stopped jobs,
1753 running background jobs and all jobs whose status has changed and have not
1754 been reported by the shell."
1755 """
1756 # NOTE: A job is a background process or pipeline.
1757 #
1758 # echo hi | wc -l -- this starts two processes. Wait for TWO
1759 # echo hi | wc -l & -- this starts a process which starts two processes
1760 # Wait for ONE.
1761 #
1762 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1763 # for %% and %-, PID, status, and "command".
1764 #
1765 # Every component of a pipeline is on the same line with 'jobs', but
1766 # they're separated into different lines with 'jobs -l'.
1767 #
1768 # See demo/jobs-builtin.sh
1769
1770 # $ jobs -l
1771 # [1]+ 24414 Stopped sleep 5
1772 # 24415 | sleep 5
1773 # [2] 24502 Running sleep 6
1774 # 24503 | sleep 6
1775 # 24504 | sleep 5 &
1776 # [3]- 24508 Running sleep 6
1777 # 24509 | sleep 6
1778 # 24510 | sleep 5 &
1779
1780 f = mylib.Stdout()
1781 for job_id, job in iteritems(self.jobs):
1782 # Use the %1 syntax
1783 job.DisplayJob(job_id, f, style)
1784
1785 def DebugPrint(self):
1786 # type: () -> None
1787
1788 f = mylib.Stdout()
1789 f.write('\n')
1790 f.write('[process debug info]\n')
1791
1792 for pid, proc in iteritems(self.child_procs):
1793 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1794 #p = ' |' if proc.parent_pipeline else ''
1795 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1796
1797 if len(self.debug_pipelines):
1798 f.write('\n')
1799 f.write('[pipeline debug info]\n')
1800 for pi in self.debug_pipelines:
1801 pi.DebugPrint()
1802
1803 def ListRecent(self):
1804 # type: () -> None
1805 """For jobs -n, which I think is also used in the interactive
1806 prompt."""
1807 pass
1808
1809 def NumRunning(self):
1810 # type: () -> int
1811 """Return the number of running jobs.
1812
1813 Used by 'wait' and 'wait -n'.
1814 """
1815 count = 0
1816 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1817 if job.State() == job_state_e.Running:
1818 count += 1
1819 return count
1820
1821
1822# Some WaitForOne() return values
1823W1_OK = -2 # waitpid(-1) returned
1824W1_ECHILD = -3 # no processes to wait for
1825W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1826
1827
1828class Waiter(object):
1829 """A capability to wait for processes.
1830
1831 This must be a singleton (and is because CommandEvaluator is a singleton).
1832
1833 Invariants:
1834 - Every child process is registered once
1835 - Every child process is waited for
1836
1837 Canonical example of why we need a GLOBAL waiter:
1838
1839 { sleep 3; echo 'done 3'; } &
1840 { sleep 4; echo 'done 4'; } &
1841
1842 # ... do arbitrary stuff ...
1843
1844 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1845
1846 Now when you do wait() after starting the pipeline, you might get a pipeline
1847 process OR a background process! So you have to distinguish between them.
1848 """
1849
1850 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1851 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1852 self.job_list = job_list
1853 self.exec_opts = exec_opts
1854 self.signal_safe = signal_safe
1855 self.tracer = tracer
1856 self.last_status = 127 # wait -n error code
1857
1858 def WaitForOne(self, waitpid_options=0):
1859 # type: (int) -> int
1860 """Wait until the next process returns (or maybe Ctrl-C).
1861
1862 Returns:
1863 One of these negative numbers:
1864 W1_ECHILD Nothing to wait for
1865 W1_OK Caller should keep waiting
1866 UNTRAPPED_SIGWINCH
1867 Or
1868 result > 0 Signal that waitpid() was interrupted with
1869
1870 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1871 will try again.
1872
1873 Callers:
1874 wait -n -- loop until there is one fewer process (TODO)
1875 wait -- loop until there are no processes
1876 wait $! -- loop until job state is Done (process or pipeline)
1877 Process::Wait() -- loop until Process state is done
1878 Pipeline::Wait() -- loop until Pipeline state is done
1879
1880 Comparisons:
1881 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1882 the wait builtin
1883
1884 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1885 loop while (gotsigchld), but that might be a hack for System V!
1886
1887 Should we have a cleaner API like named posix::wait_for_one() ?
1888
1889 wait_result =
1890 ECHILD -- nothing to wait for
1891 | Done(int pid, int status) -- process done
1892 | EINTR(bool sigint) -- may or may not retry
1893 """
1894 pid, status = pyos.WaitPid(waitpid_options)
1895 if pid == 0: # WNOHANG passed, and no state changes
1896 return W1_AGAIN
1897 elif pid < 0: # error case
1898 err_num = status
1899 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1900 if err_num == ECHILD:
1901 return W1_ECHILD # nothing to wait for caller should stop
1902 elif err_num == EINTR: # Bug #858 fix
1903 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1904 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1905 else:
1906 # The signature of waitpid() means this shouldn't happen
1907 raise AssertionError()
1908
1909 # All child processes are supposed to be in this dict. But this may
1910 # legitimately happen if a grandchild outlives the child (its parent).
1911 # Then it is reparented under this process, so we might receive
1912 # notification of its exit, even though we didn't start it. We can't have
1913 # any knowledge of such processes, so print a warning.
1914 if pid not in self.job_list.child_procs:
1915 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1916 return W1_OK
1917
1918 proc = self.job_list.child_procs[pid]
1919 if 0:
1920 self.job_list.DebugPrint()
1921
1922 if WIFSIGNALED(status):
1923 term_sig = WTERMSIG(status)
1924 status = 128 + term_sig
1925
1926 # Print newline after Ctrl-C.
1927 if term_sig == SIGINT:
1928 print('')
1929
1930 proc.WhenDone(pid, status)
1931
1932 elif WIFEXITED(status):
1933 status = WEXITSTATUS(status)
1934 #log('exit status: %s', status)
1935 proc.WhenDone(pid, status)
1936
1937 elif WIFSTOPPED(status):
1938 #status = WEXITSTATUS(status)
1939 stop_sig = WSTOPSIG(status)
1940
1941 print_stderr('')
1942 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1943 proc.WhenStopped(stop_sig)
1944
1945 else:
1946 raise AssertionError(status)
1947
1948 self.last_status = status # for wait -n
1949 self.tracer.OnProcessEnd(pid, status)
1950 return W1_OK
1951
1952 def PollNotifications(self):
1953 # type: () -> None
1954 """
1955 Process all pending state changes.
1956 """
1957 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1958 continue