OILS / core / process.py View on Github | oilshell.org

1960 lines, 951 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC, EEXIST
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from display import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_EXCL,
58 O_NONBLOCK,
59 O_NOCTTY,
60 O_RDONLY,
61 O_RDWR,
62 O_WRONLY,
63 O_TRUNC,
64)
65
66from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
67
68if TYPE_CHECKING:
69 from _devbuild.gen.runtime_asdl import cmd_value
70 from _devbuild.gen.syntax_asdl import command_t
71 from builtin import trap_osh
72 from core import optview
73 from core.util import _DebugFile
74 from osh.cmd_eval import CommandEvaluator
75
76NO_FD = -1
77
78# Minimum file descriptor that the shell can use. Other descriptors can be
79# directly used by user programs, e.g. exec 9>&1
80#
81# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
82# This is a compromise between bash (unlimited, but requires crazy
83# bookkeeping), and dash/zsh (10) and mksh (24)
84_SHELL_MIN_FD = 100
85
86# Style for 'jobs' builtin
87STYLE_DEFAULT = 0
88STYLE_LONG = 1
89STYLE_PID_ONLY = 2
90
91# To save on allocations in JobList::GetJobWithSpec()
92CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
93
94
95class ctx_FileCloser(object):
96
97 def __init__(self, f):
98 # type: (mylib.LineReader) -> None
99 self.f = f
100
101 def __enter__(self):
102 # type: () -> None
103 pass
104
105 def __exit__(self, type, value, traceback):
106 # type: (Any, Any, Any) -> None
107 self.f.close()
108
109
110def InitInteractiveShell():
111 # type: () -> None
112 """Called when initializing an interactive shell."""
113
114 # The shell itself should ignore Ctrl-\.
115 pyos.Sigaction(SIGQUIT, SIG_IGN)
116
117 # This prevents Ctrl-Z from suspending OSH in interactive mode.
118 pyos.Sigaction(SIGTSTP, SIG_IGN)
119
120 # More signals from
121 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
122 # (but not SIGCHLD)
123 pyos.Sigaction(SIGTTOU, SIG_IGN)
124 pyos.Sigaction(SIGTTIN, SIG_IGN)
125
126 # Register a callback to receive terminal width changes.
127 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
128
129 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
130 # read() have to handle it
131 pyos.RegisterSignalInterest(SIGWINCH)
132
133
134def SaveFd(fd):
135 # type: (int) -> int
136 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
137 return saved
138
139
140class _RedirFrame(object):
141
142 def __init__(self, saved_fd, orig_fd, forget):
143 # type: (int, int, bool) -> None
144 self.saved_fd = saved_fd
145 self.orig_fd = orig_fd
146 self.forget = forget
147
148
149class _FdFrame(object):
150
151 def __init__(self):
152 # type: () -> None
153 self.saved = [] # type: List[_RedirFrame]
154 self.need_wait = [] # type: List[Process]
155
156 def Forget(self):
157 # type: () -> None
158 """For exec 1>&2."""
159 for rf in reversed(self.saved):
160 if rf.saved_fd != NO_FD and rf.forget:
161 posix.close(rf.saved_fd)
162
163 del self.saved[:] # like list.clear() in Python 3.3
164 del self.need_wait[:]
165
166 def __repr__(self):
167 # type: () -> str
168 return '<_FdFrame %s>' % self.saved
169
170
171class FdState(object):
172 """File descriptor state for the current process.
173
174 For example, you can do 'myfunc > out.txt' without forking. Child
175 processes inherit our state.
176 """
177
178 def __init__(
179 self,
180 errfmt, # type: ui.ErrorFormatter
181 job_control, # type: JobControl
182 job_list, # type: JobList
183 mem, # type: state.Mem
184 tracer, # type: Optional[dev.Tracer]
185 waiter, # type: Optional[Waiter]
186 exec_opts, # type: optview.Exec
187 ):
188 # type: (...) -> None
189 """
190 Args:
191 errfmt: for errors
192 job_list: For keeping track of _HereDocWriterThunk
193 """
194 self.errfmt = errfmt
195 self.job_control = job_control
196 self.job_list = job_list
197 self.cur_frame = _FdFrame() # for the top level
198 self.stack = [self.cur_frame]
199 self.mem = mem
200 self.tracer = tracer
201 self.waiter = waiter
202 self.exec_opts = exec_opts
203
204 def Open(self, path):
205 # type: (str) -> mylib.LineReader
206 """Opens a path for read, but moves it out of the reserved 3-9 fd
207 range.
208
209 Returns:
210 A Python file object. The caller is responsible for Close().
211
212 Raises:
213 IOError or OSError if the path can't be found. (This is Python-induced wart)
214 """
215 fd_mode = O_RDONLY
216 f = self._Open(path, 'r', fd_mode)
217
218 # Hacky downcast
219 return cast('mylib.LineReader', f)
220
221 # used for util.DebugFile
222 def OpenForWrite(self, path):
223 # type: (str) -> mylib.Writer
224 fd_mode = O_CREAT | O_RDWR
225 f = self._Open(path, 'w', fd_mode)
226
227 # Hacky downcast
228 return cast('mylib.Writer', f)
229
230 def _Open(self, path, c_mode, fd_mode):
231 # type: (str, str, int) -> IO[str]
232 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
233
234 # Immediately move it to a new location
235 new_fd = SaveFd(fd)
236 posix.close(fd)
237
238 # Return a Python file handle
239 f = posix.fdopen(new_fd, c_mode) # may raise IOError
240 return f
241
242 def _WriteFdToMem(self, fd_name, fd):
243 # type: (str, int) -> None
244 if self.mem:
245 # setvar, not setref
246 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
247 value.Str(str(fd)))
248
249 def _ReadFdFromMem(self, fd_name):
250 # type: (str) -> int
251 val = self.mem.GetValue(fd_name)
252 if val.tag() == value_e.Str:
253 try:
254 return int(cast(value.Str, val).s)
255 except ValueError:
256 return NO_FD
257 return NO_FD
258
259 def _PushSave(self, fd):
260 # type: (int) -> bool
261 """Save fd to a new location and remember to restore it later."""
262 #log('---- _PushSave %s', fd)
263 ok = True
264 try:
265 new_fd = SaveFd(fd)
266 except (IOError, OSError) as e:
267 ok = False
268 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
269 # open.
270 # This seems to be ignored in dash too in savefd()?
271 if e.errno != EBADF:
272 raise
273 if ok:
274 posix.close(fd)
275 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
276 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
277 else:
278 # if we got EBADF, we still need to close the original on Pop()
279 self._PushClose(fd)
280
281 return ok
282
283 def _PushDup(self, fd1, blame_loc):
284 # type: (int, redir_loc_t) -> int
285 """Save fd2 in a higher range, and dup fd1 onto fd2.
286
287 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
288 """
289 UP_loc = blame_loc
290 if blame_loc.tag() == redir_loc_e.VarName:
291 fd2_name = cast(redir_loc.VarName, UP_loc).name
292 try:
293 # F_DUPFD: GREATER than range
294 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
295 except (IOError, OSError) as e:
296 if e.errno == EBADF:
297 print_stderr('F_DUPFD fd %d: %s' %
298 (fd1, pyutil.strerror(e)))
299 return NO_FD
300 else:
301 raise # this redirect failed
302
303 self._WriteFdToMem(fd2_name, new_fd)
304
305 elif blame_loc.tag() == redir_loc_e.Fd:
306 fd2 = cast(redir_loc.Fd, UP_loc).fd
307
308 if fd1 == fd2:
309 # The user could have asked for it to be open on descriptor 3, but open()
310 # already returned 3, e.g. echo 3>out.txt
311 return NO_FD
312
313 # Check the validity of fd1 before _PushSave(fd2)
314 try:
315 fcntl_.fcntl(fd1, F_GETFD)
316 except (IOError, OSError) as e:
317 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
318 raise
319
320 need_restore = self._PushSave(fd2)
321
322 #log('==== dup2 %s %s\n' % (fd1, fd2))
323 try:
324 posix.dup2(fd1, fd2)
325 except (IOError, OSError) as e:
326 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
327 print_stderr('dup2(%d, %d): %s' %
328 (fd1, fd2, pyutil.strerror(e)))
329
330 # Restore and return error
331 if need_restore:
332 rf = self.cur_frame.saved.pop()
333 posix.dup2(rf.saved_fd, rf.orig_fd)
334 posix.close(rf.saved_fd)
335
336 raise # this redirect failed
337
338 new_fd = fd2
339
340 else:
341 raise AssertionError()
342
343 return new_fd
344
345 def _PushCloseFd(self, blame_loc):
346 # type: (redir_loc_t) -> bool
347 """For 2>&-"""
348 # exec {fd}>&- means close the named descriptor
349
350 UP_loc = blame_loc
351 if blame_loc.tag() == redir_loc_e.VarName:
352 fd_name = cast(redir_loc.VarName, UP_loc).name
353 fd = self._ReadFdFromMem(fd_name)
354 if fd == NO_FD:
355 return False
356
357 elif blame_loc.tag() == redir_loc_e.Fd:
358 fd = cast(redir_loc.Fd, UP_loc).fd
359
360 else:
361 raise AssertionError()
362
363 self._PushSave(fd)
364
365 return True
366
367 def _PushClose(self, fd):
368 # type: (int) -> None
369 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
370
371 def _PushWait(self, proc):
372 # type: (Process) -> None
373 self.cur_frame.need_wait.append(proc)
374
375 def _ApplyRedirect(self, r):
376 # type: (RedirValue) -> None
377 arg = r.arg
378 UP_arg = arg
379 with tagswitch(arg) as case:
380
381 if case(redirect_arg_e.Path):
382 arg = cast(redirect_arg.Path, UP_arg)
383 # noclobber flag is OR'd with other flags when allowed
384 noclobber_mode = O_EXCL if self.exec_opts.noclobber() else 0
385 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
386 # NOTE: This is different than >| because it respects noclobber, but
387 # that option is almost never used. See test/wild.sh.
388 mode = O_CREAT | O_WRONLY | O_TRUNC | noclobber_mode
389 elif r.op_id == Id.Redir_Clobber: # >|
390 mode = O_CREAT | O_WRONLY | O_TRUNC
391 elif r.op_id in (Id.Redir_DGreat,
392 Id.Redir_AndDGreat): # >> &>>
393 mode = O_CREAT | O_WRONLY | O_APPEND | noclobber_mode
394 elif r.op_id == Id.Redir_Less: # <
395 mode = O_RDONLY
396 elif r.op_id == Id.Redir_LessGreat: # <>
397 mode = O_CREAT | O_RDWR
398 else:
399 raise NotImplementedError(r.op_id)
400
401 # NOTE: 0666 is affected by umask, all shells use it.
402 try:
403 open_fd = posix.open(arg.filename, mode, 0o666)
404 except (IOError, OSError) as e:
405 if e.errno == EEXIST and self.exec_opts.noclobber():
406 extra = ' (noclobber)'
407 else:
408 extra = ''
409 self.errfmt.Print_(
410 "Can't open %r: %s%s" %
411 (arg.filename, pyutil.strerror(e), extra),
412 blame_loc=r.op_loc)
413 raise # redirect failed
414
415 new_fd = self._PushDup(open_fd, r.loc)
416 if new_fd != NO_FD:
417 posix.close(open_fd)
418
419 # Now handle &> and &>> and their variants. These pairs are the same:
420 #
421 # stdout_stderr.py &> out-err.txt
422 # stdout_stderr.py > out-err.txt 2>&1
423 #
424 # stdout_stderr.py 3&> out-err.txt
425 # stdout_stderr.py 3> out-err.txt 2>&3
426 #
427 # Ditto for {fd}> and {fd}&>
428
429 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
430 self._PushDup(new_fd, redir_loc.Fd(2))
431
432 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
433 arg = cast(redirect_arg.CopyFd, UP_arg)
434
435 if r.op_id == Id.Redir_GreatAnd: # 1>&2
436 self._PushDup(arg.target_fd, r.loc)
437
438 elif r.op_id == Id.Redir_LessAnd: # 0<&5
439 # The only difference between >& and <& is the default file
440 # descriptor argument.
441 self._PushDup(arg.target_fd, r.loc)
442
443 else:
444 raise NotImplementedError()
445
446 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
447 arg = cast(redirect_arg.MoveFd, UP_arg)
448 new_fd = self._PushDup(arg.target_fd, r.loc)
449 if new_fd != NO_FD:
450 posix.close(arg.target_fd)
451
452 UP_loc = r.loc
453 if r.loc.tag() == redir_loc_e.Fd:
454 fd = cast(redir_loc.Fd, UP_loc).fd
455 else:
456 fd = NO_FD
457
458 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
459
460 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
461 self._PushCloseFd(r.loc)
462
463 elif case(redirect_arg_e.HereDoc):
464 arg = cast(redirect_arg.HereDoc, UP_arg)
465
466 # NOTE: Do these descriptors have to be moved out of the range 0-9?
467 read_fd, write_fd = posix.pipe()
468
469 self._PushDup(read_fd, r.loc) # stdin is now the pipe
470
471 # We can't close like we do in the filename case above? The writer can
472 # get a "broken pipe".
473 self._PushClose(read_fd)
474
475 thunk = _HereDocWriterThunk(write_fd, arg.body)
476
477 # Use PIPE_SIZE to save a process in the case of small here
478 # docs, which are the common case. (dash does this.)
479
480 # Note: could instrument this to see how often it happens.
481 # Though strace -ff can also work.
482 start_process = len(arg.body) > 4096
483 #start_process = True
484
485 if start_process:
486 here_proc = Process(thunk, self.job_control, self.job_list,
487 self.tracer)
488
489 # NOTE: we could close the read pipe here, but it doesn't really
490 # matter because we control the code.
491 here_proc.StartProcess(trace.HereDoc)
492 #log('Started %s as %d', here_proc, pid)
493 self._PushWait(here_proc)
494
495 # Now that we've started the child, close it in the parent.
496 posix.close(write_fd)
497
498 else:
499 posix.write(write_fd, arg.body)
500 posix.close(write_fd)
501
502 def Push(self, redirects, err_out):
503 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
504 """Apply a group of redirects and remember to undo them."""
505
506 #log('> fd_state.Push %s', redirects)
507 new_frame = _FdFrame()
508 self.stack.append(new_frame)
509 self.cur_frame = new_frame
510
511 for r in redirects:
512 #log('apply %s', r)
513 with ui.ctx_Location(self.errfmt, r.op_loc):
514 try:
515 self._ApplyRedirect(r)
516 except (IOError, OSError) as e:
517 err_out.append(e)
518 # This can fail too
519 self.Pop(err_out)
520 return # for bad descriptor, etc.
521
522 def PushStdinFromPipe(self, r):
523 # type: (int) -> bool
524 """Save the current stdin and make it come from descriptor 'r'.
525
526 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
527 semantics of
528
529 echo foo | read line; echo $line
530 """
531 new_frame = _FdFrame()
532 self.stack.append(new_frame)
533 self.cur_frame = new_frame
534
535 self._PushDup(r, redir_loc.Fd(0))
536 return True
537
538 def Pop(self, err_out):
539 # type: (List[error.IOError_OSError]) -> None
540 frame = self.stack.pop()
541 #log('< Pop %s', frame)
542 for rf in reversed(frame.saved):
543 if rf.saved_fd == NO_FD:
544 #log('Close %d', orig)
545 try:
546 posix.close(rf.orig_fd)
547 except (IOError, OSError) as e:
548 err_out.append(e)
549 log('Error closing descriptor %d: %s', rf.orig_fd,
550 pyutil.strerror(e))
551 return
552 else:
553 try:
554 posix.dup2(rf.saved_fd, rf.orig_fd)
555 except (IOError, OSError) as e:
556 err_out.append(e)
557 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
558 pyutil.strerror(e))
559 #log('fd state:')
560 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
561 return
562 posix.close(rf.saved_fd)
563 #log('dup2 %s %s', saved, orig)
564
565 # Wait for here doc processes to finish.
566 for proc in frame.need_wait:
567 unused_status = proc.Wait(self.waiter)
568
569 def MakePermanent(self):
570 # type: () -> None
571 self.cur_frame.Forget()
572
573
574class ChildStateChange(object):
575
576 def __init__(self):
577 # type: () -> None
578 """Empty constructor for mycpp."""
579 pass
580
581 def Apply(self):
582 # type: () -> None
583 raise NotImplementedError()
584
585 def ApplyFromParent(self, proc):
586 # type: (Process) -> None
587 """Noop for all state changes other than SetPgid for mycpp."""
588 pass
589
590
591class StdinFromPipe(ChildStateChange):
592
593 def __init__(self, pipe_read_fd, w):
594 # type: (int, int) -> None
595 self.r = pipe_read_fd
596 self.w = w
597
598 def __repr__(self):
599 # type: () -> str
600 return '<StdinFromPipe %d %d>' % (self.r, self.w)
601
602 def Apply(self):
603 # type: () -> None
604 posix.dup2(self.r, 0)
605 posix.close(self.r) # close after dup
606
607 posix.close(self.w) # we're reading from the pipe, not writing
608 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
609
610
611class StdoutToPipe(ChildStateChange):
612
613 def __init__(self, r, pipe_write_fd):
614 # type: (int, int) -> None
615 self.r = r
616 self.w = pipe_write_fd
617
618 def __repr__(self):
619 # type: () -> str
620 return '<StdoutToPipe %d %d>' % (self.r, self.w)
621
622 def Apply(self):
623 # type: () -> None
624 posix.dup2(self.w, 1)
625 posix.close(self.w) # close after dup
626
627 posix.close(self.r) # we're writing to the pipe, not reading
628 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
629
630
631INVALID_PGID = -1
632# argument to setpgid() that means the process is its own leader
633OWN_LEADER = 0
634
635
636class SetPgid(ChildStateChange):
637
638 def __init__(self, pgid, tracer):
639 # type: (int, dev.Tracer) -> None
640 self.pgid = pgid
641 self.tracer = tracer
642
643 def Apply(self):
644 # type: () -> None
645 try:
646 posix.setpgid(0, self.pgid)
647 except (IOError, OSError) as e:
648 self.tracer.OtherMessage(
649 'osh: child %d failed to set its process group to %d: %s' %
650 (posix.getpid(), self.pgid, pyutil.strerror(e)))
651
652 def ApplyFromParent(self, proc):
653 # type: (Process) -> None
654 try:
655 posix.setpgid(proc.pid, self.pgid)
656 except (IOError, OSError) as e:
657 self.tracer.OtherMessage(
658 'osh: parent failed to set process group for PID %d to %d: %s'
659 % (proc.pid, self.pgid, pyutil.strerror(e)))
660
661
662class ExternalProgram(object):
663 """The capability to execute an external program like 'ls'."""
664
665 def __init__(
666 self,
667 hijack_shebang, # type: str
668 fd_state, # type: FdState
669 errfmt, # type: ui.ErrorFormatter
670 debug_f, # type: _DebugFile
671 ):
672 # type: (...) -> None
673 """
674 Args:
675 hijack_shebang: The path of an interpreter to run instead of the one
676 specified in the shebang line. May be empty.
677 """
678 self.hijack_shebang = hijack_shebang
679 self.fd_state = fd_state
680 self.errfmt = errfmt
681 self.debug_f = debug_f
682
683 def Exec(self, argv0_path, cmd_val, environ):
684 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
685 """Execute a program and exit this process.
686
687 Called by: ls / exec ls / ( ls / )
688 """
689 probe('process', 'ExternalProgram_Exec', argv0_path)
690 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
691 True)
692 assert False, "This line should never execute" # NO RETURN
693
694 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
695 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
696 if len(self.hijack_shebang):
697 opened = True
698 try:
699 f = self.fd_state.Open(argv0_path)
700 except (IOError, OSError) as e:
701 opened = False
702
703 if opened:
704 with ctx_FileCloser(f):
705 # Test if the shebang looks like a shell. TODO: The file might be
706 # binary with no newlines, so read 80 bytes instead of readline().
707
708 #line = f.read(80) # type: ignore # TODO: fix this
709 line = f.readline()
710
711 if match.ShouldHijack(line):
712 h_argv = [self.hijack_shebang, argv0_path]
713 h_argv.extend(argv[1:])
714 argv = h_argv
715 argv0_path = self.hijack_shebang
716 self.debug_f.writeln('Hijacked: %s' % argv0_path)
717 else:
718 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
719 pass
720
721 try:
722 posix.execve(argv0_path, argv, environ)
723 except (IOError, OSError) as e:
724 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
725 if e.errno == ENOEXEC and should_retry:
726 new_argv = ['/bin/sh', argv0_path]
727 new_argv.extend(argv[1:])
728 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
729 # NO RETURN
730
731 # Would be nice: when the path is relative and ENOENT: print PWD and do
732 # spelling correction?
733
734 self.errfmt.Print_(
735 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
736 argv0_loc)
737
738 # POSIX mentions 126 and 127 for two specific errors. The rest are
739 # unspecified.
740 #
741 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
742 if e.errno == EACCES:
743 status = 126
744 elif e.errno == ENOENT:
745 # TODO: most shells print 'command not found', rather than strerror()
746 # == "No such file or directory". That's better because it's at the
747 # end of the path search, and we're never searching for a directory.
748 status = 127
749 else:
750 # dash uses 2, but we use that for parse errors. This seems to be
751 # consistent with mksh and zsh.
752 status = 127
753
754 posix._exit(status)
755 # NO RETURN
756
757
758class Thunk(object):
759 """Abstract base class for things runnable in another process."""
760
761 def __init__(self):
762 # type: () -> None
763 """Empty constructor for mycpp."""
764 pass
765
766 def Run(self):
767 # type: () -> None
768 """Returns a status code."""
769 raise NotImplementedError()
770
771 def UserString(self):
772 # type: () -> str
773 """Display for the 'jobs' list."""
774 raise NotImplementedError()
775
776 def __repr__(self):
777 # type: () -> str
778 return self.UserString()
779
780
781class ExternalThunk(Thunk):
782 """An external executable."""
783
784 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
785 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
786 self.ext_prog = ext_prog
787 self.argv0_path = argv0_path
788 self.cmd_val = cmd_val
789 self.environ = environ
790
791 def UserString(self):
792 # type: () -> str
793
794 # NOTE: This is the format the Tracer uses.
795 # bash displays sleep $n & (code)
796 # but OSH displays sleep 1 & (argv array)
797 # We could switch the former but I'm not sure it's necessary.
798 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
799 return '[process] %s' % ' '.join(tmp)
800
801 def Run(self):
802 # type: () -> None
803 """An ExternalThunk is run in parent for the exec builtin."""
804 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
805
806
807class SubProgramThunk(Thunk):
808 """A subprogram that can be executed in another process."""
809
810 def __init__(self,
811 cmd_ev,
812 node,
813 trap_state,
814 multi_trace,
815 inherit_errexit=True):
816 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool) -> None
817 self.cmd_ev = cmd_ev
818 self.node = node
819 self.trap_state = trap_state
820 self.multi_trace = multi_trace
821 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
822
823 def UserString(self):
824 # type: () -> str
825
826 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
827 # TODO: Extract SPIDS from node to display source? Note that
828 # CompoundStatus also has locations of each pipeline component; see
829 # Executor.RunPipeline()
830 thunk_str = ui.CommandType(self.node)
831 return '[subprog] %s' % thunk_str
832
833 def Run(self):
834 # type: () -> None
835 #self.errfmt.OneLineErrExit() # don't quote code in child processes
836 probe('process', 'SubProgramThunk_Run')
837
838 # TODO: break circular dep. Bit flags could go in ASDL or headers.
839 from osh import cmd_eval
840
841 # signal handlers aren't inherited
842 self.trap_state.ClearForSubProgram()
843
844 # NOTE: may NOT return due to exec().
845 if not self.inherit_errexit:
846 self.cmd_ev.mutable_opts.DisableErrExit()
847 try:
848 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
849 self.cmd_ev.ExecuteAndCatch(self.node, cmd_flags=cmd_eval.Optimize)
850 status = self.cmd_ev.LastStatus()
851 # NOTE: We ignore the is_fatal return value. The user should set -o
852 # errexit so failures in subprocesses cause failures in the parent.
853 except util.UserExit as e:
854 status = e.status
855
856 # Handle errors in a subshell. These two cases are repeated from main()
857 # and the core/completion.py hook.
858 except KeyboardInterrupt:
859 print('')
860 status = 130 # 128 + 2
861 except (IOError, OSError) as e:
862 print_stderr('oils I/O error (subprogram): %s' %
863 pyutil.strerror(e))
864 status = 2
865
866 # If ProcessInit() doesn't turn off buffering, this is needed before
867 # _exit()
868 pyos.FlushStdout()
869
870 self.multi_trace.WriteDumps()
871
872 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
873 # gets called in BOTH processes.
874 # The crash dump seems to be unaffected.
875 posix._exit(status)
876
877
878class _HereDocWriterThunk(Thunk):
879 """Write a here doc to one end of a pipe.
880
881 May be be executed in either a child process or the main shell
882 process.
883 """
884
885 def __init__(self, w, body_str):
886 # type: (int, str) -> None
887 self.w = w
888 self.body_str = body_str
889
890 def UserString(self):
891 # type: () -> str
892
893 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
894 # shells don't have this problem because they use temp files! That's a bit
895 # unfortunate.
896 return '[here doc writer]'
897
898 def Run(self):
899 # type: () -> None
900 """do_exit: For small pipelines."""
901 probe('process', 'HereDocWriterThunk_Run')
902 #log('Writing %r', self.body_str)
903 posix.write(self.w, self.body_str)
904 #log('Wrote %r', self.body_str)
905 posix.close(self.w)
906 #log('Closed %d', self.w)
907
908 posix._exit(0)
909
910
911class Job(object):
912 """Interface for both Process and Pipeline.
913
914 They both can be put in the background and waited on.
915
916 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
917
918 sleep 1 | sleep 2 &
919
920 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
921 not a JOB ID.
922 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
923 - The process group leader (setpgid) is the FIRST PID.
924 - It's also %1 or %+. The last job started.
925 """
926
927 def __init__(self):
928 # type: () -> None
929 # Initial state with & or Ctrl-Z is Running.
930 self.state = job_state_e.Running
931 self.job_id = -1
932 self.in_background = False
933
934 def DisplayJob(self, job_id, f, style):
935 # type: (int, mylib.Writer, int) -> None
936 raise NotImplementedError()
937
938 def State(self):
939 # type: () -> job_state_t
940 return self.state
941
942 def ProcessGroupId(self):
943 # type: () -> int
944 """Return the process group ID associated with this job."""
945 raise NotImplementedError()
946
947 def JobWait(self, waiter):
948 # type: (Waiter) -> wait_status_t
949 """Wait for this process/pipeline to be stopped or finished."""
950 raise NotImplementedError()
951
952 def SetBackground(self):
953 # type: () -> None
954 """Record that this job is running in the background."""
955 self.in_background = True
956
957 def SetForeground(self):
958 # type: () -> None
959 """Record that this job is running in the foreground."""
960 self.in_background = False
961
962
963class Process(Job):
964 """A process to run.
965
966 TODO: Should we make it clear that this is a FOREGROUND process? A
967 background process is wrapped in a "job". It is unevaluated.
968
969 It provides an API to manipulate file descriptor state in parent and child.
970 """
971
972 def __init__(self, thunk, job_control, job_list, tracer):
973 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
974 """
975 Args:
976 thunk: Thunk instance
977 job_list: for process bookkeeping
978 """
979 Job.__init__(self)
980 assert isinstance(thunk, Thunk), thunk
981 self.thunk = thunk
982 self.job_control = job_control
983 self.job_list = job_list
984 self.tracer = tracer
985
986 # For pipelines
987 self.parent_pipeline = None # type: Pipeline
988 self.state_changes = [] # type: List[ChildStateChange]
989 self.close_r = -1
990 self.close_w = -1
991
992 self.pid = -1
993 self.status = -1
994
995 def Init_ParentPipeline(self, pi):
996 # type: (Pipeline) -> None
997 """For updating PIPESTATUS."""
998 self.parent_pipeline = pi
999
1000 def __repr__(self):
1001 # type: () -> str
1002
1003 # note: be wary of infinite mutual recursion
1004 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
1005 #return '<Process %s%s>' % (self.thunk, s)
1006 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
1007
1008 def ProcessGroupId(self):
1009 # type: () -> int
1010 """Returns the group ID of this process."""
1011 # This should only ever be called AFTER the process has started
1012 assert self.pid != -1
1013 if self.parent_pipeline:
1014 # XXX: Maybe we should die here instead? Unclear if this branch
1015 # should even be reachable with the current builtins.
1016 return self.parent_pipeline.ProcessGroupId()
1017
1018 return self.pid
1019
1020 def DisplayJob(self, job_id, f, style):
1021 # type: (int, mylib.Writer, int) -> None
1022 if job_id == -1:
1023 job_id_str = ' '
1024 else:
1025 job_id_str = '%%%d' % job_id
1026 if style == STYLE_PID_ONLY:
1027 f.write('%d\n' % self.pid)
1028 else:
1029 f.write('%s %d %7s ' %
1030 (job_id_str, self.pid, _JobStateStr(self.state)))
1031 f.write(self.thunk.UserString())
1032 f.write('\n')
1033
1034 def AddStateChange(self, s):
1035 # type: (ChildStateChange) -> None
1036 self.state_changes.append(s)
1037
1038 def AddPipeToClose(self, r, w):
1039 # type: (int, int) -> None
1040 self.close_r = r
1041 self.close_w = w
1042
1043 def MaybeClosePipe(self):
1044 # type: () -> None
1045 if self.close_r != -1:
1046 posix.close(self.close_r)
1047 posix.close(self.close_w)
1048
1049 def StartProcess(self, why):
1050 # type: (trace_t) -> int
1051 """Start this process with fork(), handling redirects."""
1052 pid = posix.fork()
1053 if pid < 0:
1054 # When does this happen?
1055 e_die('Fatal error in posix.fork()')
1056
1057 elif pid == 0: # child
1058 # Note: this happens in BOTH interactive and non-interactive shells.
1059 # We technically don't need to do most of it in non-interactive, since we
1060 # did not change state in InitInteractiveShell().
1061
1062 for st in self.state_changes:
1063 st.Apply()
1064
1065 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1066 # shouldn't have this.
1067 # https://docs.python.org/2/library/signal.html
1068 # See Python/pythonrun.c.
1069 pyos.Sigaction(SIGPIPE, SIG_DFL)
1070
1071 # Respond to Ctrl-\ (core dump)
1072 pyos.Sigaction(SIGQUIT, SIG_DFL)
1073
1074 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1075 # foreground because suspending them is difficult with our 'lastpipe'
1076 # semantics.
1077 pid = posix.getpid()
1078 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1079 pyos.Sigaction(SIGTSTP, SIG_DFL)
1080
1081 # More signals from
1082 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1083 # (but not SIGCHLD)
1084 pyos.Sigaction(SIGTTOU, SIG_DFL)
1085 pyos.Sigaction(SIGTTIN, SIG_DFL)
1086
1087 self.tracer.OnNewProcess(pid)
1088 # clear foreground pipeline for subshells
1089 self.thunk.Run()
1090 # Never returns
1091
1092 #log('STARTED process %s, pid = %d', self, pid)
1093 self.tracer.OnProcessStart(pid, why)
1094
1095 # Class invariant: after the process is started, it stores its PID.
1096 self.pid = pid
1097
1098 # SetPgid needs to be applied from the child and the parent to avoid
1099 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1100 for st in self.state_changes:
1101 st.ApplyFromParent(self)
1102
1103 # Program invariant: We keep track of every child process!
1104 self.job_list.AddChildProcess(pid, self)
1105
1106 return pid
1107
1108 def Wait(self, waiter):
1109 # type: (Waiter) -> int
1110 """Wait for this process to finish."""
1111 while self.state == job_state_e.Running:
1112 # Only return if there's nothing to wait for. Keep waiting if we were
1113 # interrupted with a signal.
1114 if waiter.WaitForOne() == W1_ECHILD:
1115 break
1116
1117 assert self.status >= 0, self.status
1118 return self.status
1119
1120 def JobWait(self, waiter):
1121 # type: (Waiter) -> wait_status_t
1122 # wait builtin can be interrupted
1123 while self.state == job_state_e.Running:
1124 result = waiter.WaitForOne()
1125
1126 if result >= 0: # signal
1127 return wait_status.Cancelled(result)
1128
1129 if result == W1_ECHILD:
1130 break
1131
1132 return wait_status.Proc(self.status)
1133
1134 def WhenStopped(self, stop_sig):
1135 # type: (int) -> None
1136
1137 # 128 is a shell thing
1138 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1139 self.status = 128 + stop_sig
1140 self.state = job_state_e.Stopped
1141
1142 if self.job_id == -1:
1143 # This process was started in the foreground
1144 self.job_list.AddJob(self)
1145
1146 if not self.in_background:
1147 self.job_control.MaybeTakeTerminal()
1148 self.SetBackground()
1149
1150 def WhenDone(self, pid, status):
1151 # type: (int, int) -> None
1152 """Called by the Waiter when this Process finishes."""
1153
1154 #log('WhenDone %d %d', pid, status)
1155 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1156 self.status = status
1157 self.state = job_state_e.Done
1158 if self.parent_pipeline:
1159 self.parent_pipeline.WhenDone(pid, status)
1160 else:
1161 if self.job_id != -1:
1162 # Job might have been brought to the foreground after being
1163 # assigned a job ID.
1164 if self.in_background:
1165 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1166
1167 self.job_list.RemoveJob(self.job_id)
1168
1169 self.job_list.RemoveChildProcess(self.pid)
1170
1171 if not self.in_background:
1172 self.job_control.MaybeTakeTerminal()
1173
1174 def RunProcess(self, waiter, why):
1175 # type: (Waiter, trace_t) -> int
1176 """Run this process synchronously."""
1177 self.StartProcess(why)
1178 # ShellExecutor might be calling this for the last part of a pipeline.
1179 if self.parent_pipeline is None:
1180 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1181 # calling getpgid()?
1182 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1183 return self.Wait(waiter)
1184
1185
1186class ctx_Pipe(object):
1187
1188 def __init__(self, fd_state, fd, err_out):
1189 # type: (FdState, int, List[error.IOError_OSError]) -> None
1190 fd_state.PushStdinFromPipe(fd)
1191 self.fd_state = fd_state
1192 self.err_out = err_out
1193
1194 def __enter__(self):
1195 # type: () -> None
1196 pass
1197
1198 def __exit__(self, type, value, traceback):
1199 # type: (Any, Any, Any) -> None
1200 self.fd_state.Pop(self.err_out)
1201
1202
1203class Pipeline(Job):
1204 """A pipeline of processes to run.
1205
1206 Cases we handle:
1207
1208 foo | bar
1209 $(foo | bar)
1210 foo | bar | read v
1211 """
1212
1213 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1214 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1215 Job.__init__(self)
1216 self.job_control = job_control
1217 self.job_list = job_list
1218 self.tracer = tracer
1219
1220 self.procs = [] # type: List[Process]
1221 self.pids = [] # type: List[int] # pids in order
1222 self.pipe_status = [] # type: List[int] # status in order
1223 self.status = -1 # for 'wait' jobs
1224
1225 self.pgid = INVALID_PGID
1226
1227 # Optional for foreground
1228 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1229 self.last_pipe = None # type: Tuple[int, int]
1230
1231 self.sigpipe_status_ok = sigpipe_status_ok
1232
1233 def ProcessGroupId(self):
1234 # type: () -> int
1235 """Returns the group ID of this pipeline."""
1236 return self.pgid
1237
1238 def DisplayJob(self, job_id, f, style):
1239 # type: (int, mylib.Writer, int) -> None
1240 if style == STYLE_PID_ONLY:
1241 f.write('%d\n' % self.procs[0].pid)
1242 else:
1243 # Note: this is STYLE_LONG.
1244 for i, proc in enumerate(self.procs):
1245 if i == 0: # show job ID for first element in pipeline
1246 job_id_str = '%%%d' % job_id
1247 else:
1248 job_id_str = ' ' # 2 spaces
1249
1250 f.write('%s %d %7s ' %
1251 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1252 f.write(proc.thunk.UserString())
1253 f.write('\n')
1254
1255 def DebugPrint(self):
1256 # type: () -> None
1257 print('Pipeline in state %s' % _JobStateStr(self.state))
1258 if mylib.PYTHON: # %s for Process not allowed in C++
1259 for proc in self.procs:
1260 print(' proc %s' % proc)
1261 _, last_node = self.last_thunk
1262 print(' last %s' % last_node)
1263 print(' pipe_status %s' % self.pipe_status)
1264
1265 def Add(self, p):
1266 # type: (Process) -> None
1267 """Append a process to the pipeline."""
1268 if len(self.procs) == 0:
1269 self.procs.append(p)
1270 return
1271
1272 r, w = posix.pipe()
1273 #log('pipe for %s: %d %d', p, r, w)
1274 prev = self.procs[-1]
1275
1276 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1277 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1278
1279 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1280
1281 self.procs.append(p)
1282
1283 def AddLast(self, thunk):
1284 # type: (Tuple[CommandEvaluator, command_t]) -> None
1285 """Append the last noden to the pipeline.
1286
1287 This is run in the CURRENT process. It is OPTIONAL, because
1288 pipelines in the background are run uniformly.
1289 """
1290 self.last_thunk = thunk
1291
1292 assert len(self.procs) != 0
1293
1294 r, w = posix.pipe()
1295 prev = self.procs[-1]
1296 prev.AddStateChange(StdoutToPipe(r, w))
1297
1298 self.last_pipe = (r, w) # So we can connect it to last_thunk
1299
1300 def StartPipeline(self, waiter):
1301 # type: (Waiter) -> None
1302
1303 # If we are creating a pipeline in a subshell or we aren't running with job
1304 # control, our children should remain in our inherited process group.
1305 # the pipelines's group ID.
1306 if self.job_control.Enabled():
1307 self.pgid = OWN_LEADER # first process in pipeline is the leader
1308
1309 for i, proc in enumerate(self.procs):
1310 if self.pgid != INVALID_PGID:
1311 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1312
1313 # Figure out the pid
1314 pid = proc.StartProcess(trace.PipelinePart)
1315 if i == 0 and self.pgid != INVALID_PGID:
1316 # Mimic bash and use the PID of the FIRST process as the group for the
1317 # whole pipeline.
1318 self.pgid = pid
1319
1320 self.pids.append(pid)
1321 self.pipe_status.append(-1) # uninitialized
1322
1323 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1324 # It can't be done at the end; otherwise processes will have descriptors
1325 # from non-adjacent pipes.
1326 proc.MaybeClosePipe()
1327
1328 if self.last_thunk:
1329 self.pipe_status.append(-1) # for self.last_thunk
1330
1331 def LastPid(self):
1332 # type: () -> int
1333 """For the odd $! variable.
1334
1335 It would be better if job IDs or PGIDs were used consistently.
1336 """
1337 return self.pids[-1]
1338
1339 def Wait(self, waiter):
1340 # type: (Waiter) -> List[int]
1341 """Wait for this pipeline to finish."""
1342
1343 assert self.procs, "no procs for Wait()"
1344 # waitpid(-1) zero or more times
1345 while self.state == job_state_e.Running:
1346 # Keep waiting until there's nothing to wait for.
1347 if waiter.WaitForOne() == W1_ECHILD:
1348 break
1349
1350 return self.pipe_status
1351
1352 def JobWait(self, waiter):
1353 # type: (Waiter) -> wait_status_t
1354 """Called by 'wait' builtin, e.g. 'wait %1'."""
1355 # wait builtin can be interrupted
1356 assert self.procs, "no procs for Wait()"
1357 while self.state == job_state_e.Running:
1358 result = waiter.WaitForOne()
1359
1360 if result >= 0: # signal
1361 return wait_status.Cancelled(result)
1362
1363 if result == W1_ECHILD:
1364 break
1365
1366 return wait_status.Pipeline(self.pipe_status)
1367
1368 def RunLastPart(self, waiter, fd_state):
1369 # type: (Waiter, FdState) -> List[int]
1370 """Run this pipeline synchronously (foreground pipeline).
1371
1372 Returns:
1373 pipe_status (list of integers).
1374 """
1375 assert len(self.pids) == len(self.procs)
1376
1377 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1378 from osh import cmd_eval
1379
1380 # This is tcsetpgrp()
1381 # TODO: fix race condition -- I believe the first process could have
1382 # stopped already, and thus getpgid() will fail
1383 self.job_control.MaybeGiveTerminal(self.pgid)
1384
1385 # Run the last part of the pipeline IN PARALLEL with other processes. It
1386 # may or may not fork:
1387 # echo foo | read line # no fork, the builtin runs in THIS shell process
1388 # ls | wc -l # fork for 'wc'
1389
1390 cmd_ev, last_node = self.last_thunk
1391
1392 assert self.last_pipe is not None
1393 r, w = self.last_pipe # set in AddLast()
1394 posix.close(w) # we will not write here
1395
1396 # Fix lastpipe / job control / DEBUG trap interaction
1397 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1398
1399 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1400 # a pipeline.
1401 cmd_flags |= cmd_eval.NoErrTrap
1402
1403 io_errors = [] # type: List[error.IOError_OSError]
1404 with ctx_Pipe(fd_state, r, io_errors):
1405 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1406
1407 if len(io_errors):
1408 e_die('Error setting up last part of pipeline: %s' %
1409 pyutil.strerror(io_errors[0]))
1410
1411 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1412 # /dev/urandom | sleep 1' will never get SIGPIPE.
1413 posix.close(r)
1414
1415 self.pipe_status[-1] = cmd_ev.LastStatus()
1416 if self.AllDone():
1417 self.state = job_state_e.Done
1418
1419 #log('pipestatus before all have finished = %s', self.pipe_status)
1420 return self.Wait(waiter)
1421
1422 def AllDone(self):
1423 # type: () -> bool
1424
1425 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1426 for status in self.pipe_status:
1427 if status == -1:
1428 return False
1429 return True
1430
1431 def WhenDone(self, pid, status):
1432 # type: (int, int) -> None
1433 """Called by Process.WhenDone."""
1434 #log('Pipeline WhenDone %d %d', pid, status)
1435 i = self.pids.index(pid)
1436 assert i != -1, 'Unexpected PID %d' % pid
1437
1438 if status == 141 and self.sigpipe_status_ok:
1439 status = 0
1440
1441 self.job_list.RemoveChildProcess(pid)
1442 self.pipe_status[i] = status
1443 if self.AllDone():
1444 if self.job_id != -1:
1445 # Job might have been brought to the foreground after being
1446 # assigned a job ID.
1447 if self.in_background:
1448 print_stderr('[%d] Done PGID %d' %
1449 (self.job_id, self.pids[0]))
1450
1451 self.job_list.RemoveJob(self.job_id)
1452
1453 # status of pipeline is status of last process
1454 self.status = self.pipe_status[-1]
1455 self.state = job_state_e.Done
1456 if not self.in_background:
1457 self.job_control.MaybeTakeTerminal()
1458
1459
1460def _JobStateStr(i):
1461 # type: (job_state_t) -> str
1462 return job_state_str(i)[10:] # remove 'job_state.'
1463
1464
1465def _GetTtyFd():
1466 # type: () -> int
1467 """Returns -1 if stdio is not a TTY."""
1468 try:
1469 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1470 except (IOError, OSError) as e:
1471 return -1
1472
1473
1474class ctx_TerminalControl(object):
1475
1476 def __init__(self, job_control, errfmt):
1477 # type: (JobControl, ui.ErrorFormatter) -> None
1478 job_control.InitJobControl()
1479 self.job_control = job_control
1480 self.errfmt = errfmt
1481
1482 def __enter__(self):
1483 # type: () -> None
1484 pass
1485
1486 def __exit__(self, type, value, traceback):
1487 # type: (Any, Any, Any) -> None
1488
1489 # Return the TTY to the original owner before exiting.
1490 try:
1491 self.job_control.MaybeReturnTerminal()
1492 except error.FatalRuntime as e:
1493 # Don't abort the shell on error, just print a message.
1494 self.errfmt.PrettyPrintError(e)
1495
1496
1497class JobControl(object):
1498 """Interface to setpgid(), tcsetpgrp(), etc."""
1499
1500 def __init__(self):
1501 # type: () -> None
1502
1503 # The main shell's PID and group ID.
1504 self.shell_pid = -1
1505 self.shell_pgid = -1
1506
1507 # The fd of the controlling tty. Set to -1 when job control is disabled.
1508 self.shell_tty_fd = -1
1509
1510 # For giving the terminal back to our parent before exiting (if not a login
1511 # shell).
1512 self.original_tty_pgid = -1
1513
1514 def InitJobControl(self):
1515 # type: () -> None
1516 self.shell_pid = posix.getpid()
1517 orig_shell_pgid = posix.getpgid(0)
1518 self.shell_pgid = orig_shell_pgid
1519 self.shell_tty_fd = _GetTtyFd()
1520
1521 # If we aren't the leader of our process group, create a group and mark
1522 # ourselves as the leader.
1523 if self.shell_pgid != self.shell_pid:
1524 try:
1525 posix.setpgid(self.shell_pid, self.shell_pid)
1526 self.shell_pgid = self.shell_pid
1527 except (IOError, OSError) as e:
1528 self.shell_tty_fd = -1
1529
1530 if self.shell_tty_fd != -1:
1531 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1532
1533 # If stdio is a TTY, put the shell's process group in the foreground.
1534 try:
1535 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1536 except (IOError, OSError) as e:
1537 # We probably aren't in the session leader's process group. Disable job
1538 # control.
1539 self.shell_tty_fd = -1
1540 self.shell_pgid = orig_shell_pgid
1541 posix.setpgid(self.shell_pid, self.shell_pgid)
1542
1543 def Enabled(self):
1544 # type: () -> bool
1545
1546 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1547 # think.
1548 curr_pid = posix.getpid()
1549 # Only the main shell should bother with job control functions.
1550 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1551
1552 # TODO: This isn't a PID. This is a process group ID?
1553 #
1554 # What should the table look like?
1555 #
1556 # Do we need the last PID? I don't know why bash prints that. Probably so
1557 # you can do wait $!
1558 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1559 #
1560 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1561 # job_id is just an integer. This is sort of lame.
1562 #
1563 # [job_id, flag, pgid, job_state, node]
1564
1565 def MaybeGiveTerminal(self, pgid):
1566 # type: (int) -> None
1567 """If stdio is a TTY, move the given process group to the
1568 foreground."""
1569 if not self.Enabled():
1570 # Only call tcsetpgrp when job control is enabled.
1571 return
1572
1573 try:
1574 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1575 except (IOError, OSError) as e:
1576 e_die('osh: Failed to move process group %d to foreground: %s' %
1577 (pgid, pyutil.strerror(e)))
1578
1579 def MaybeTakeTerminal(self):
1580 # type: () -> None
1581 """If stdio is a TTY, return the main shell's process group to the
1582 foreground."""
1583 self.MaybeGiveTerminal(self.shell_pgid)
1584
1585 def MaybeReturnTerminal(self):
1586 # type: () -> None
1587 """Called before the shell exits."""
1588 self.MaybeGiveTerminal(self.original_tty_pgid)
1589
1590
1591class JobList(object):
1592 """Global list of jobs, used by a few builtins."""
1593
1594 def __init__(self):
1595 # type: () -> None
1596
1597 # job_id -> Job instance
1598 self.jobs = {} # type: Dict[int, Job]
1599
1600 # pid -> Process. This is for STOP notification.
1601 self.child_procs = {} # type: Dict[int, Process]
1602 self.debug_pipelines = [] # type: List[Pipeline]
1603
1604 # Counter used to assign IDs to jobs. It is incremented every time a job
1605 # is created. Once all active jobs are done it is reset to 1. I'm not
1606 # sure if this reset behavior is mandated by POSIX, but other shells do
1607 # it, so we mimic for the sake of compatibility.
1608 self.job_id = 1
1609
1610 def AddJob(self, job):
1611 # type: (Job) -> int
1612 """Add a background job to the list.
1613
1614 A job is either a Process or Pipeline. You can resume a job with 'fg',
1615 kill it with 'kill', etc.
1616
1617 Two cases:
1618
1619 1. async jobs: sleep 5 | sleep 4 &
1620 2. stopped jobs: sleep 5; then Ctrl-Z
1621 """
1622 job_id = self.job_id
1623 self.jobs[job_id] = job
1624 job.job_id = job_id
1625 self.job_id += 1
1626 return job_id
1627
1628 def RemoveJob(self, job_id):
1629 # type: (int) -> None
1630 """Process and Pipeline can call this."""
1631 mylib.dict_erase(self.jobs, job_id)
1632
1633 if len(self.jobs) == 0:
1634 self.job_id = 1
1635
1636 def AddChildProcess(self, pid, proc):
1637 # type: (int, Process) -> None
1638 """Every child process should be added here as soon as we know its PID.
1639
1640 When the Waiter gets an EXITED or STOPPED notification, we need
1641 to know about it so 'jobs' can work.
1642 """
1643 self.child_procs[pid] = proc
1644
1645 def RemoveChildProcess(self, pid):
1646 # type: (int) -> None
1647 """Remove the child process with the given PID."""
1648 mylib.dict_erase(self.child_procs, pid)
1649
1650 if mylib.PYTHON:
1651
1652 def AddPipeline(self, pi):
1653 # type: (Pipeline) -> None
1654 """For debugging only."""
1655 self.debug_pipelines.append(pi)
1656
1657 def ProcessFromPid(self, pid):
1658 # type: (int) -> Process
1659 """For wait $PID.
1660
1661 There's no way to wait for a pipeline with a PID. That uses job
1662 syntax, e.g. %1. Not a great interface.
1663 """
1664 return self.child_procs.get(pid)
1665
1666 def GetCurrentAndPreviousJobs(self):
1667 # type: () -> Tuple[Optional[Job], Optional[Job]]
1668 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1669
1670 See the POSIX specification for the `jobs` builtin for details:
1671 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1672
1673 IMPORTANT NOTE: This method assumes that the jobs list will not change
1674 during its execution! This assumption holds for now because we only ever
1675 update the jobs list from the main loop after WaitPid() informs us of a
1676 change. If we implement `set -b` and install a signal handler for
1677 SIGCHLD we should be careful to synchronize it with this function. The
1678 unsafety of mutating GC data structures from a signal handler should
1679 make this a non-issue, but if bugs related to this appear this note may
1680 be helpful...
1681 """
1682 # Split all active jobs by state and sort each group by decreasing job
1683 # ID to approximate newness.
1684 stopped_jobs = [] # type: List[Job]
1685 running_jobs = [] # type: List[Job]
1686 for i in xrange(0, self.job_id):
1687 job = self.jobs.get(i, None)
1688 if not job:
1689 continue
1690
1691 if job.state == job_state_e.Stopped:
1692 stopped_jobs.append(job)
1693
1694 elif job.state == job_state_e.Running:
1695 running_jobs.append(job)
1696
1697 current = None # type: Optional[Job]
1698 previous = None # type: Optional[Job]
1699 # POSIX says: If there is any suspended job, then the current job shall
1700 # be a suspended job. If there are at least two suspended jobs, then the
1701 # previous job also shall be a suspended job.
1702 #
1703 # So, we will only return running jobs from here if there are no recent
1704 # stopped jobs.
1705 if len(stopped_jobs) > 0:
1706 current = stopped_jobs.pop()
1707
1708 if len(stopped_jobs) > 0:
1709 previous = stopped_jobs.pop()
1710
1711 if len(running_jobs) > 0 and not current:
1712 current = running_jobs.pop()
1713
1714 if len(running_jobs) > 0 and not previous:
1715 previous = running_jobs.pop()
1716
1717 if not previous:
1718 previous = current
1719
1720 return current, previous
1721
1722 def GetJobWithSpec(self, job_spec):
1723 # type: (str) -> Optional[Job]
1724 """Parse the given job spec and return the matching job. If there is no
1725 matching job, this function returns None.
1726
1727 See the POSIX spec for the `jobs` builtin for details about job specs:
1728 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1729 """
1730 if job_spec in CURRENT_JOB_SPECS:
1731 current, _ = self.GetCurrentAndPreviousJobs()
1732 return current
1733
1734 if job_spec == '%-':
1735 _, previous = self.GetCurrentAndPreviousJobs()
1736 return previous
1737
1738 # TODO: Add support for job specs based on prefixes of process argv.
1739 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1740 if m is not None:
1741 assert len(m) == 2
1742 job_id = int(m[1])
1743 if job_id in self.jobs:
1744 return self.jobs[job_id]
1745
1746 return None
1747
1748 def DisplayJobs(self, style):
1749 # type: (int) -> None
1750 """Used by the 'jobs' builtin.
1751
1752 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1753
1754 "By default, the jobs utility shall display the status of all stopped jobs,
1755 running background jobs and all jobs whose status has changed and have not
1756 been reported by the shell."
1757 """
1758 # NOTE: A job is a background process or pipeline.
1759 #
1760 # echo hi | wc -l -- this starts two processes. Wait for TWO
1761 # echo hi | wc -l & -- this starts a process which starts two processes
1762 # Wait for ONE.
1763 #
1764 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1765 # for %% and %-, PID, status, and "command".
1766 #
1767 # Every component of a pipeline is on the same line with 'jobs', but
1768 # they're separated into different lines with 'jobs -l'.
1769 #
1770 # See demo/jobs-builtin.sh
1771
1772 # $ jobs -l
1773 # [1]+ 24414 Stopped sleep 5
1774 # 24415 | sleep 5
1775 # [2] 24502 Running sleep 6
1776 # 24503 | sleep 6
1777 # 24504 | sleep 5 &
1778 # [3]- 24508 Running sleep 6
1779 # 24509 | sleep 6
1780 # 24510 | sleep 5 &
1781
1782 f = mylib.Stdout()
1783 for job_id, job in iteritems(self.jobs):
1784 # Use the %1 syntax
1785 job.DisplayJob(job_id, f, style)
1786
1787 def DebugPrint(self):
1788 # type: () -> None
1789
1790 f = mylib.Stdout()
1791 f.write('\n')
1792 f.write('[process debug info]\n')
1793
1794 for pid, proc in iteritems(self.child_procs):
1795 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1796 #p = ' |' if proc.parent_pipeline else ''
1797 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1798
1799 if len(self.debug_pipelines):
1800 f.write('\n')
1801 f.write('[pipeline debug info]\n')
1802 for pi in self.debug_pipelines:
1803 pi.DebugPrint()
1804
1805 def ListRecent(self):
1806 # type: () -> None
1807 """For jobs -n, which I think is also used in the interactive
1808 prompt."""
1809 pass
1810
1811 def NumRunning(self):
1812 # type: () -> int
1813 """Return the number of running jobs.
1814
1815 Used by 'wait' and 'wait -n'.
1816 """
1817 count = 0
1818 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1819 if job.State() == job_state_e.Running:
1820 count += 1
1821 return count
1822
1823
1824# Some WaitForOne() return values
1825W1_OK = -2 # waitpid(-1) returned
1826W1_ECHILD = -3 # no processes to wait for
1827W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1828
1829
1830class Waiter(object):
1831 """A capability to wait for processes.
1832
1833 This must be a singleton (and is because CommandEvaluator is a singleton).
1834
1835 Invariants:
1836 - Every child process is registered once
1837 - Every child process is waited for
1838
1839 Canonical example of why we need a GLOBAL waiter:
1840
1841 { sleep 3; echo 'done 3'; } &
1842 { sleep 4; echo 'done 4'; } &
1843
1844 # ... do arbitrary stuff ...
1845
1846 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1847
1848 Now when you do wait() after starting the pipeline, you might get a pipeline
1849 process OR a background process! So you have to distinguish between them.
1850 """
1851
1852 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1853 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1854 self.job_list = job_list
1855 self.exec_opts = exec_opts
1856 self.signal_safe = signal_safe
1857 self.tracer = tracer
1858 self.last_status = 127 # wait -n error code
1859
1860 def WaitForOne(self, waitpid_options=0):
1861 # type: (int) -> int
1862 """Wait until the next process returns (or maybe Ctrl-C).
1863
1864 Returns:
1865 One of these negative numbers:
1866 W1_ECHILD Nothing to wait for
1867 W1_OK Caller should keep waiting
1868 UNTRAPPED_SIGWINCH
1869 Or
1870 result > 0 Signal that waitpid() was interrupted with
1871
1872 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1873 will try again.
1874
1875 Callers:
1876 wait -n -- loop until there is one fewer process (TODO)
1877 wait -- loop until there are no processes
1878 wait $! -- loop until job state is Done (process or pipeline)
1879 Process::Wait() -- loop until Process state is done
1880 Pipeline::Wait() -- loop until Pipeline state is done
1881
1882 Comparisons:
1883 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1884 the wait builtin
1885
1886 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1887 loop while (gotsigchld), but that might be a hack for System V!
1888
1889 Should we have a cleaner API like named posix::wait_for_one() ?
1890
1891 wait_result =
1892 ECHILD -- nothing to wait for
1893 | Done(int pid, int status) -- process done
1894 | EINTR(bool sigint) -- may or may not retry
1895 """
1896 pid, status = pyos.WaitPid(waitpid_options)
1897 if pid == 0: # WNOHANG passed, and no state changes
1898 return W1_AGAIN
1899 elif pid < 0: # error case
1900 err_num = status
1901 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1902 if err_num == ECHILD:
1903 return W1_ECHILD # nothing to wait for caller should stop
1904 elif err_num == EINTR: # Bug #858 fix
1905 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1906 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1907 else:
1908 # The signature of waitpid() means this shouldn't happen
1909 raise AssertionError()
1910
1911 # All child processes are supposed to be in this dict. But this may
1912 # legitimately happen if a grandchild outlives the child (its parent).
1913 # Then it is reparented under this process, so we might receive
1914 # notification of its exit, even though we didn't start it. We can't have
1915 # any knowledge of such processes, so print a warning.
1916 if pid not in self.job_list.child_procs:
1917 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1918 return W1_OK
1919
1920 proc = self.job_list.child_procs[pid]
1921 if 0:
1922 self.job_list.DebugPrint()
1923
1924 if WIFSIGNALED(status):
1925 term_sig = WTERMSIG(status)
1926 status = 128 + term_sig
1927
1928 # Print newline after Ctrl-C.
1929 if term_sig == SIGINT:
1930 print('')
1931
1932 proc.WhenDone(pid, status)
1933
1934 elif WIFEXITED(status):
1935 status = WEXITSTATUS(status)
1936 #log('exit status: %s', status)
1937 proc.WhenDone(pid, status)
1938
1939 elif WIFSTOPPED(status):
1940 #status = WEXITSTATUS(status)
1941 stop_sig = WSTOPSIG(status)
1942
1943 print_stderr('')
1944 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1945 proc.WhenStopped(stop_sig)
1946
1947 else:
1948 raise AssertionError(status)
1949
1950 self.last_status = status # for wait -n
1951 self.tracer.OnProcessEnd(pid, status)
1952 return W1_OK
1953
1954 def PollNotifications(self):
1955 # type: () -> None
1956 """
1957 Process all pending state changes.
1958 """
1959 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1960 continue