OILS / core / process.py View on Github | oilshell.org

1960 lines, 950 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC, EEXIST
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from display import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_EXCL,
58 O_NONBLOCK,
59 O_NOCTTY,
60 O_RDONLY,
61 O_RDWR,
62 O_WRONLY,
63 O_TRUNC,
64)
65
66from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
67
68if TYPE_CHECKING:
69 from _devbuild.gen.runtime_asdl import cmd_value
70 from _devbuild.gen.syntax_asdl import command_t
71 from builtin import trap_osh
72 from core import optview
73 from core.util import _DebugFile
74 from osh.cmd_eval import CommandEvaluator
75
76NO_FD = -1
77
78# Minimum file descriptor that the shell can use. Other descriptors can be
79# directly used by user programs, e.g. exec 9>&1
80#
81# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
82# This is a compromise between bash (unlimited, but requires crazy
83# bookkeeping), and dash/zsh (10) and mksh (24)
84_SHELL_MIN_FD = 100
85
86# Style for 'jobs' builtin
87STYLE_DEFAULT = 0
88STYLE_LONG = 1
89STYLE_PID_ONLY = 2
90
91# To save on allocations in JobList::GetJobWithSpec()
92CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
93
94
95class ctx_FileCloser(object):
96
97 def __init__(self, f):
98 # type: (mylib.LineReader) -> None
99 self.f = f
100
101 def __enter__(self):
102 # type: () -> None
103 pass
104
105 def __exit__(self, type, value, traceback):
106 # type: (Any, Any, Any) -> None
107 self.f.close()
108
109
110def InitInteractiveShell():
111 # type: () -> None
112 """Called when initializing an interactive shell."""
113
114 # The shell itself should ignore Ctrl-\.
115 pyos.Sigaction(SIGQUIT, SIG_IGN)
116
117 # This prevents Ctrl-Z from suspending OSH in interactive mode.
118 pyos.Sigaction(SIGTSTP, SIG_IGN)
119
120 # More signals from
121 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
122 # (but not SIGCHLD)
123 pyos.Sigaction(SIGTTOU, SIG_IGN)
124 pyos.Sigaction(SIGTTIN, SIG_IGN)
125
126 # Register a callback to receive terminal width changes.
127 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
128
129 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
130 # read() have to handle it
131 pyos.RegisterSignalInterest(SIGWINCH)
132
133
134def SaveFd(fd):
135 # type: (int) -> int
136 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
137 return saved
138
139
140class _RedirFrame(object):
141
142 def __init__(self, saved_fd, orig_fd, forget):
143 # type: (int, int, bool) -> None
144 self.saved_fd = saved_fd
145 self.orig_fd = orig_fd
146 self.forget = forget
147
148
149class _FdFrame(object):
150
151 def __init__(self):
152 # type: () -> None
153 self.saved = [] # type: List[_RedirFrame]
154 self.need_wait = [] # type: List[Process]
155
156 def Forget(self):
157 # type: () -> None
158 """For exec 1>&2."""
159 for rf in reversed(self.saved):
160 if rf.saved_fd != NO_FD and rf.forget:
161 posix.close(rf.saved_fd)
162
163 del self.saved[:] # like list.clear() in Python 3.3
164 del self.need_wait[:]
165
166 def __repr__(self):
167 # type: () -> str
168 return '<_FdFrame %s>' % self.saved
169
170
171class FdState(object):
172 """File descriptor state for the current process.
173
174 For example, you can do 'myfunc > out.txt' without forking. Child
175 processes inherit our state.
176 """
177
178 def __init__(
179 self,
180 errfmt, # type: ui.ErrorFormatter
181 job_control, # type: JobControl
182 job_list, # type: JobList
183 mem, # type: state.Mem
184 tracer, # type: Optional[dev.Tracer]
185 waiter, # type: Optional[Waiter]
186 exec_opts, # type: optview.Exec
187 ):
188 # type: (...) -> None
189 """
190 Args:
191 errfmt: for errors
192 job_list: For keeping track of _HereDocWriterThunk
193 """
194 self.errfmt = errfmt
195 self.job_control = job_control
196 self.job_list = job_list
197 self.cur_frame = _FdFrame() # for the top level
198 self.stack = [self.cur_frame]
199 self.mem = mem
200 self.tracer = tracer
201 self.waiter = waiter
202 self.exec_opts = exec_opts
203
204 def Open(self, path):
205 # type: (str) -> mylib.LineReader
206 """Opens a path for read, but moves it out of the reserved 3-9 fd
207 range.
208
209 Returns:
210 A Python file object. The caller is responsible for Close().
211
212 Raises:
213 IOError or OSError if the path can't be found. (This is Python-induced wart)
214 """
215 fd_mode = O_RDONLY
216 f = self._Open(path, 'r', fd_mode)
217
218 # Hacky downcast
219 return cast('mylib.LineReader', f)
220
221 # used for util.DebugFile
222 def OpenForWrite(self, path):
223 # type: (str) -> mylib.Writer
224 fd_mode = O_CREAT | O_RDWR
225 f = self._Open(path, 'w', fd_mode)
226
227 # Hacky downcast
228 return cast('mylib.Writer', f)
229
230 def _Open(self, path, c_mode, fd_mode):
231 # type: (str, str, int) -> IO[str]
232 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
233
234 # Immediately move it to a new location
235 new_fd = SaveFd(fd)
236 posix.close(fd)
237
238 # Return a Python file handle
239 f = posix.fdopen(new_fd, c_mode) # may raise IOError
240 return f
241
242 def _WriteFdToMem(self, fd_name, fd):
243 # type: (str, int) -> None
244 if self.mem:
245 # setvar, not setref
246 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
247 value.Str(str(fd)))
248
249 def _ReadFdFromMem(self, fd_name):
250 # type: (str) -> int
251 val = self.mem.GetValue(fd_name)
252 if val.tag() == value_e.Str:
253 try:
254 return int(cast(value.Str, val).s)
255 except ValueError:
256 return NO_FD
257 return NO_FD
258
259 def _PushSave(self, fd):
260 # type: (int) -> bool
261 """Save fd to a new location and remember to restore it later."""
262 #log('---- _PushSave %s', fd)
263 ok = True
264 try:
265 new_fd = SaveFd(fd)
266 except (IOError, OSError) as e:
267 ok = False
268 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
269 # open.
270 # This seems to be ignored in dash too in savefd()?
271 if e.errno != EBADF:
272 raise
273 if ok:
274 posix.close(fd)
275 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
276 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
277 else:
278 # if we got EBADF, we still need to close the original on Pop()
279 self._PushClose(fd)
280
281 return ok
282
283 def _PushDup(self, fd1, blame_loc):
284 # type: (int, redir_loc_t) -> int
285 """Save fd2 in a higher range, and dup fd1 onto fd2.
286
287 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
288 """
289 UP_loc = blame_loc
290 if blame_loc.tag() == redir_loc_e.VarName:
291 fd2_name = cast(redir_loc.VarName, UP_loc).name
292 try:
293 # F_DUPFD: GREATER than range
294 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
295 except (IOError, OSError) as e:
296 if e.errno == EBADF:
297 print_stderr('F_DUPFD fd %d: %s' %
298 (fd1, pyutil.strerror(e)))
299 return NO_FD
300 else:
301 raise # this redirect failed
302
303 self._WriteFdToMem(fd2_name, new_fd)
304
305 elif blame_loc.tag() == redir_loc_e.Fd:
306 fd2 = cast(redir_loc.Fd, UP_loc).fd
307
308 if fd1 == fd2:
309 # The user could have asked for it to be open on descriptor 3, but open()
310 # already returned 3, e.g. echo 3>out.txt
311 return NO_FD
312
313 # Check the validity of fd1 before _PushSave(fd2)
314 try:
315 fcntl_.fcntl(fd1, F_GETFD)
316 except (IOError, OSError) as e:
317 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
318 raise
319
320 need_restore = self._PushSave(fd2)
321
322 #log('==== dup2 %s %s\n' % (fd1, fd2))
323 try:
324 posix.dup2(fd1, fd2)
325 except (IOError, OSError) as e:
326 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
327 print_stderr('dup2(%d, %d): %s' %
328 (fd1, fd2, pyutil.strerror(e)))
329
330 # Restore and return error
331 if need_restore:
332 rf = self.cur_frame.saved.pop()
333 posix.dup2(rf.saved_fd, rf.orig_fd)
334 posix.close(rf.saved_fd)
335
336 raise # this redirect failed
337
338 new_fd = fd2
339
340 else:
341 raise AssertionError()
342
343 return new_fd
344
345 def _PushCloseFd(self, blame_loc):
346 # type: (redir_loc_t) -> bool
347 """For 2>&-"""
348 # exec {fd}>&- means close the named descriptor
349
350 UP_loc = blame_loc
351 if blame_loc.tag() == redir_loc_e.VarName:
352 fd_name = cast(redir_loc.VarName, UP_loc).name
353 fd = self._ReadFdFromMem(fd_name)
354 if fd == NO_FD:
355 return False
356
357 elif blame_loc.tag() == redir_loc_e.Fd:
358 fd = cast(redir_loc.Fd, UP_loc).fd
359
360 else:
361 raise AssertionError()
362
363 self._PushSave(fd)
364
365 return True
366
367 def _PushClose(self, fd):
368 # type: (int) -> None
369 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
370
371 def _PushWait(self, proc):
372 # type: (Process) -> None
373 self.cur_frame.need_wait.append(proc)
374
375 def _ApplyRedirect(self, r):
376 # type: (RedirValue) -> None
377 arg = r.arg
378 UP_arg = arg
379 with tagswitch(arg) as case:
380
381 if case(redirect_arg_e.Path):
382 arg = cast(redirect_arg.Path, UP_arg)
383 # noclobber flag is OR'd with other flags when allowed
384 noclobber_mode = O_EXCL if self.exec_opts.noclobber() else 0
385 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
386 # NOTE: This is different than >| because it respects noclobber, but
387 # that option is almost never used. See test/wild.sh.
388 mode = O_CREAT | O_WRONLY | O_TRUNC | noclobber_mode
389 elif r.op_id == Id.Redir_Clobber: # >|
390 mode = O_CREAT | O_WRONLY | O_TRUNC
391 elif r.op_id in (Id.Redir_DGreat,
392 Id.Redir_AndDGreat): # >> &>>
393 mode = O_CREAT | O_WRONLY | O_APPEND | noclobber_mode
394 elif r.op_id == Id.Redir_Less: # <
395 mode = O_RDONLY
396 elif r.op_id == Id.Redir_LessGreat: # <>
397 mode = O_CREAT | O_RDWR
398 else:
399 raise NotImplementedError(r.op_id)
400
401 # NOTE: 0666 is affected by umask, all shells use it.
402 try:
403 open_fd = posix.open(arg.filename, mode, 0o666)
404 except (IOError, OSError) as e:
405 if e.errno == EEXIST and self.exec_opts.noclobber():
406 extra = ' (noclobber)'
407 else:
408 extra = ''
409 self.errfmt.Print_(
410 "Can't open %r: %s%s" %
411 (arg.filename, pyutil.strerror(e), extra),
412 blame_loc=r.op_loc)
413 raise # redirect failed
414
415 new_fd = self._PushDup(open_fd, r.loc)
416 if new_fd != NO_FD:
417 posix.close(open_fd)
418
419 # Now handle &> and &>> and their variants. These pairs are the same:
420 #
421 # stdout_stderr.py &> out-err.txt
422 # stdout_stderr.py > out-err.txt 2>&1
423 #
424 # stdout_stderr.py 3&> out-err.txt
425 # stdout_stderr.py 3> out-err.txt 2>&3
426 #
427 # Ditto for {fd}> and {fd}&>
428
429 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
430 self._PushDup(new_fd, redir_loc.Fd(2))
431
432 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
433 arg = cast(redirect_arg.CopyFd, UP_arg)
434
435 if r.op_id == Id.Redir_GreatAnd: # 1>&2
436 self._PushDup(arg.target_fd, r.loc)
437
438 elif r.op_id == Id.Redir_LessAnd: # 0<&5
439 # The only difference between >& and <& is the default file
440 # descriptor argument.
441 self._PushDup(arg.target_fd, r.loc)
442
443 else:
444 raise NotImplementedError()
445
446 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
447 arg = cast(redirect_arg.MoveFd, UP_arg)
448 new_fd = self._PushDup(arg.target_fd, r.loc)
449 if new_fd != NO_FD:
450 posix.close(arg.target_fd)
451
452 UP_loc = r.loc
453 if r.loc.tag() == redir_loc_e.Fd:
454 fd = cast(redir_loc.Fd, UP_loc).fd
455 else:
456 fd = NO_FD
457
458 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
459
460 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
461 self._PushCloseFd(r.loc)
462
463 elif case(redirect_arg_e.HereDoc):
464 arg = cast(redirect_arg.HereDoc, UP_arg)
465
466 # NOTE: Do these descriptors have to be moved out of the range 0-9?
467 read_fd, write_fd = posix.pipe()
468
469 self._PushDup(read_fd, r.loc) # stdin is now the pipe
470
471 # We can't close like we do in the filename case above? The writer can
472 # get a "broken pipe".
473 self._PushClose(read_fd)
474
475 thunk = _HereDocWriterThunk(write_fd, arg.body)
476
477 # Use PIPE_SIZE to save a process in the case of small here
478 # docs, which are the common case. (dash does this.)
479
480 # Note: could instrument this to see how often it happens.
481 # Though strace -ff can also work.
482 start_process = len(arg.body) > 4096
483 #start_process = True
484
485 if start_process:
486 here_proc = Process(thunk, self.job_control, self.job_list,
487 self.tracer)
488
489 # NOTE: we could close the read pipe here, but it doesn't really
490 # matter because we control the code.
491 here_proc.StartProcess(trace.HereDoc)
492 #log('Started %s as %d', here_proc, pid)
493 self._PushWait(here_proc)
494
495 # Now that we've started the child, close it in the parent.
496 posix.close(write_fd)
497
498 else:
499 posix.write(write_fd, arg.body)
500 posix.close(write_fd)
501
502 def Push(self, redirects, err_out):
503 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
504 """Apply a group of redirects and remember to undo them."""
505
506 #log('> fd_state.Push %s', redirects)
507 new_frame = _FdFrame()
508 self.stack.append(new_frame)
509 self.cur_frame = new_frame
510
511 for r in redirects:
512 #log('apply %s', r)
513 with ui.ctx_Location(self.errfmt, r.op_loc):
514 try:
515 self._ApplyRedirect(r)
516 except (IOError, OSError) as e:
517 err_out.append(e)
518 # This can fail too
519 self.Pop(err_out)
520 return # for bad descriptor, etc.
521
522 def PushStdinFromPipe(self, r):
523 # type: (int) -> bool
524 """Save the current stdin and make it come from descriptor 'r'.
525
526 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
527 semantics of
528
529 echo foo | read line; echo $line
530 """
531 new_frame = _FdFrame()
532 self.stack.append(new_frame)
533 self.cur_frame = new_frame
534
535 self._PushDup(r, redir_loc.Fd(0))
536 return True
537
538 def Pop(self, err_out):
539 # type: (List[error.IOError_OSError]) -> None
540 frame = self.stack.pop()
541 #log('< Pop %s', frame)
542 for rf in reversed(frame.saved):
543 if rf.saved_fd == NO_FD:
544 #log('Close %d', orig)
545 try:
546 posix.close(rf.orig_fd)
547 except (IOError, OSError) as e:
548 err_out.append(e)
549 log('Error closing descriptor %d: %s', rf.orig_fd,
550 pyutil.strerror(e))
551 return
552 else:
553 try:
554 posix.dup2(rf.saved_fd, rf.orig_fd)
555 except (IOError, OSError) as e:
556 err_out.append(e)
557 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
558 pyutil.strerror(e))
559 #log('fd state:')
560 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
561 return
562 posix.close(rf.saved_fd)
563 #log('dup2 %s %s', saved, orig)
564
565 # Wait for here doc processes to finish.
566 for proc in frame.need_wait:
567 unused_status = proc.Wait(self.waiter)
568
569 def MakePermanent(self):
570 # type: () -> None
571 self.cur_frame.Forget()
572
573
574class ChildStateChange(object):
575
576 def __init__(self):
577 # type: () -> None
578 """Empty constructor for mycpp."""
579 pass
580
581 def Apply(self):
582 # type: () -> None
583 raise NotImplementedError()
584
585 def ApplyFromParent(self, proc):
586 # type: (Process) -> None
587 """Noop for all state changes other than SetPgid for mycpp."""
588 pass
589
590
591class StdinFromPipe(ChildStateChange):
592
593 def __init__(self, pipe_read_fd, w):
594 # type: (int, int) -> None
595 self.r = pipe_read_fd
596 self.w = w
597
598 def __repr__(self):
599 # type: () -> str
600 return '<StdinFromPipe %d %d>' % (self.r, self.w)
601
602 def Apply(self):
603 # type: () -> None
604 posix.dup2(self.r, 0)
605 posix.close(self.r) # close after dup
606
607 posix.close(self.w) # we're reading from the pipe, not writing
608 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
609
610
611class StdoutToPipe(ChildStateChange):
612
613 def __init__(self, r, pipe_write_fd):
614 # type: (int, int) -> None
615 self.r = r
616 self.w = pipe_write_fd
617
618 def __repr__(self):
619 # type: () -> str
620 return '<StdoutToPipe %d %d>' % (self.r, self.w)
621
622 def Apply(self):
623 # type: () -> None
624 posix.dup2(self.w, 1)
625 posix.close(self.w) # close after dup
626
627 posix.close(self.r) # we're writing to the pipe, not reading
628 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
629
630
631INVALID_PGID = -1
632# argument to setpgid() that means the process is its own leader
633OWN_LEADER = 0
634
635
636class SetPgid(ChildStateChange):
637
638 def __init__(self, pgid, tracer):
639 # type: (int, dev.Tracer) -> None
640 self.pgid = pgid
641 self.tracer = tracer
642
643 def Apply(self):
644 # type: () -> None
645 try:
646 posix.setpgid(0, self.pgid)
647 except (IOError, OSError) as e:
648 self.tracer.OtherMessage(
649 'osh: child %d failed to set its process group to %d: %s' %
650 (posix.getpid(), self.pgid, pyutil.strerror(e)))
651
652 def ApplyFromParent(self, proc):
653 # type: (Process) -> None
654 try:
655 posix.setpgid(proc.pid, self.pgid)
656 except (IOError, OSError) as e:
657 self.tracer.OtherMessage(
658 'osh: parent failed to set process group for PID %d to %d: %s'
659 % (proc.pid, self.pgid, pyutil.strerror(e)))
660
661
662class ExternalProgram(object):
663 """The capability to execute an external program like 'ls'."""
664
665 def __init__(
666 self,
667 hijack_shebang, # type: str
668 fd_state, # type: FdState
669 errfmt, # type: ui.ErrorFormatter
670 debug_f, # type: _DebugFile
671 ):
672 # type: (...) -> None
673 """
674 Args:
675 hijack_shebang: The path of an interpreter to run instead of the one
676 specified in the shebang line. May be empty.
677 """
678 self.hijack_shebang = hijack_shebang
679 self.fd_state = fd_state
680 self.errfmt = errfmt
681 self.debug_f = debug_f
682
683 def Exec(self, argv0_path, cmd_val, environ):
684 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
685 """Execute a program and exit this process.
686
687 Called by: ls / exec ls / ( ls / )
688 """
689 probe('process', 'ExternalProgram_Exec', argv0_path)
690 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
691 True)
692 assert False, "This line should never execute" # NO RETURN
693
694 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
695 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
696 if len(self.hijack_shebang):
697 opened = True
698 try:
699 f = self.fd_state.Open(argv0_path)
700 except (IOError, OSError) as e:
701 opened = False
702
703 if opened:
704 with ctx_FileCloser(f):
705 # Test if the shebang looks like a shell. TODO: The file might be
706 # binary with no newlines, so read 80 bytes instead of readline().
707
708 #line = f.read(80) # type: ignore # TODO: fix this
709 line = f.readline()
710
711 if match.ShouldHijack(line):
712 h_argv = [self.hijack_shebang, argv0_path]
713 h_argv.extend(argv[1:])
714 argv = h_argv
715 argv0_path = self.hijack_shebang
716 self.debug_f.writeln('Hijacked: %s' % argv0_path)
717 else:
718 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
719 pass
720
721 try:
722 posix.execve(argv0_path, argv, environ)
723 except (IOError, OSError) as e:
724 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
725 if e.errno == ENOEXEC and should_retry:
726 new_argv = ['/bin/sh', argv0_path]
727 new_argv.extend(argv[1:])
728 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
729 # NO RETURN
730
731 # Would be nice: when the path is relative and ENOENT: print PWD and do
732 # spelling correction?
733
734 self.errfmt.Print_(
735 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
736 argv0_loc)
737
738 # POSIX mentions 126 and 127 for two specific errors. The rest are
739 # unspecified.
740 #
741 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
742 if e.errno == EACCES:
743 status = 126
744 elif e.errno == ENOENT:
745 # TODO: most shells print 'command not found', rather than strerror()
746 # == "No such file or directory". That's better because it's at the
747 # end of the path search, and we're never searching for a directory.
748 status = 127
749 else:
750 # dash uses 2, but we use that for parse errors. This seems to be
751 # consistent with mksh and zsh.
752 status = 127
753
754 posix._exit(status)
755 # NO RETURN
756
757
758class Thunk(object):
759 """Abstract base class for things runnable in another process."""
760
761 def __init__(self):
762 # type: () -> None
763 """Empty constructor for mycpp."""
764 pass
765
766 def Run(self):
767 # type: () -> None
768 """Returns a status code."""
769 raise NotImplementedError()
770
771 def UserString(self):
772 # type: () -> str
773 """Display for the 'jobs' list."""
774 raise NotImplementedError()
775
776 def __repr__(self):
777 # type: () -> str
778 return self.UserString()
779
780
781class ExternalThunk(Thunk):
782 """An external executable."""
783
784 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
785 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
786 self.ext_prog = ext_prog
787 self.argv0_path = argv0_path
788 self.cmd_val = cmd_val
789 self.environ = environ
790
791 def UserString(self):
792 # type: () -> str
793
794 # NOTE: This is the format the Tracer uses.
795 # bash displays sleep $n & (code)
796 # but OSH displays sleep 1 & (argv array)
797 # We could switch the former but I'm not sure it's necessary.
798 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
799 return '[process] %s' % ' '.join(tmp)
800
801 def Run(self):
802 # type: () -> None
803 """An ExternalThunk is run in parent for the exec builtin."""
804 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
805
806
807class SubProgramThunk(Thunk):
808 """A subprogram that can be executed in another process."""
809
810 def __init__(self, cmd_ev, node, trap_state, multi_trace, inherit_errexit,
811 inherit_errtrace):
812 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool, bool) -> None
813 self.cmd_ev = cmd_ev
814 self.node = node
815 self.trap_state = trap_state
816 self.multi_trace = multi_trace
817 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
818 self.inherit_errtrace = inherit_errtrace # for bash errtrace compatibility
819
820 def UserString(self):
821 # type: () -> str
822
823 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
824 # TODO: Extract SPIDS from node to display source? Note that
825 # CompoundStatus also has locations of each pipeline component; see
826 # Executor.RunPipeline()
827 thunk_str = ui.CommandType(self.node)
828 return '[subprog] %s' % thunk_str
829
830 def Run(self):
831 # type: () -> None
832 #self.errfmt.OneLineErrExit() # don't quote code in child processes
833 probe('process', 'SubProgramThunk_Run')
834
835 # TODO: break circular dep. Bit flags could go in ASDL or headers.
836 from osh import cmd_eval
837
838 # signal handlers aren't inherited
839 self.trap_state.ClearForSubProgram(self.inherit_errtrace)
840
841 # NOTE: may NOT return due to exec().
842 if not self.inherit_errexit:
843 self.cmd_ev.mutable_opts.DisableErrExit()
844 try:
845 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
846 self.cmd_ev.ExecuteAndCatch(
847 self.node,
848 cmd_eval.OptimizeSubshells | cmd_eval.MarkLastCommands)
849 status = self.cmd_ev.LastStatus()
850 # NOTE: We ignore the is_fatal return value. The user should set -o
851 # errexit so failures in subprocesses cause failures in the parent.
852 except util.UserExit as e:
853 status = e.status
854
855 # Handle errors in a subshell. These two cases are repeated from main()
856 # and the core/completion.py hook.
857 except KeyboardInterrupt:
858 print('')
859 status = 130 # 128 + 2
860 except (IOError, OSError) as e:
861 print_stderr('oils I/O error (subprogram): %s' %
862 pyutil.strerror(e))
863 status = 2
864
865 # If ProcessInit() doesn't turn off buffering, this is needed before
866 # _exit()
867 pyos.FlushStdout()
868
869 self.multi_trace.WriteDumps()
870
871 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
872 # gets called in BOTH processes.
873 # The crash dump seems to be unaffected.
874 posix._exit(status)
875
876
877class _HereDocWriterThunk(Thunk):
878 """Write a here doc to one end of a pipe.
879
880 May be be executed in either a child process or the main shell
881 process.
882 """
883
884 def __init__(self, w, body_str):
885 # type: (int, str) -> None
886 self.w = w
887 self.body_str = body_str
888
889 def UserString(self):
890 # type: () -> str
891
892 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
893 # shells don't have this problem because they use temp files! That's a bit
894 # unfortunate.
895 return '[here doc writer]'
896
897 def Run(self):
898 # type: () -> None
899 """do_exit: For small pipelines."""
900 probe('process', 'HereDocWriterThunk_Run')
901 #log('Writing %r', self.body_str)
902 posix.write(self.w, self.body_str)
903 #log('Wrote %r', self.body_str)
904 posix.close(self.w)
905 #log('Closed %d', self.w)
906
907 posix._exit(0)
908
909
910class Job(object):
911 """Interface for both Process and Pipeline.
912
913 They both can be put in the background and waited on.
914
915 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
916
917 sleep 1 | sleep 2 &
918
919 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
920 not a JOB ID.
921 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
922 - The process group leader (setpgid) is the FIRST PID.
923 - It's also %1 or %+. The last job started.
924 """
925
926 def __init__(self):
927 # type: () -> None
928 # Initial state with & or Ctrl-Z is Running.
929 self.state = job_state_e.Running
930 self.job_id = -1
931 self.in_background = False
932
933 def DisplayJob(self, job_id, f, style):
934 # type: (int, mylib.Writer, int) -> None
935 raise NotImplementedError()
936
937 def State(self):
938 # type: () -> job_state_t
939 return self.state
940
941 def ProcessGroupId(self):
942 # type: () -> int
943 """Return the process group ID associated with this job."""
944 raise NotImplementedError()
945
946 def JobWait(self, waiter):
947 # type: (Waiter) -> wait_status_t
948 """Wait for this process/pipeline to be stopped or finished."""
949 raise NotImplementedError()
950
951 def SetBackground(self):
952 # type: () -> None
953 """Record that this job is running in the background."""
954 self.in_background = True
955
956 def SetForeground(self):
957 # type: () -> None
958 """Record that this job is running in the foreground."""
959 self.in_background = False
960
961
962class Process(Job):
963 """A process to run.
964
965 TODO: Should we make it clear that this is a FOREGROUND process? A
966 background process is wrapped in a "job". It is unevaluated.
967
968 It provides an API to manipulate file descriptor state in parent and child.
969 """
970
971 def __init__(self, thunk, job_control, job_list, tracer):
972 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
973 """
974 Args:
975 thunk: Thunk instance
976 job_list: for process bookkeeping
977 """
978 Job.__init__(self)
979 assert isinstance(thunk, Thunk), thunk
980 self.thunk = thunk
981 self.job_control = job_control
982 self.job_list = job_list
983 self.tracer = tracer
984
985 # For pipelines
986 self.parent_pipeline = None # type: Pipeline
987 self.state_changes = [] # type: List[ChildStateChange]
988 self.close_r = -1
989 self.close_w = -1
990
991 self.pid = -1
992 self.status = -1
993
994 def Init_ParentPipeline(self, pi):
995 # type: (Pipeline) -> None
996 """For updating PIPESTATUS."""
997 self.parent_pipeline = pi
998
999 def __repr__(self):
1000 # type: () -> str
1001
1002 # note: be wary of infinite mutual recursion
1003 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
1004 #return '<Process %s%s>' % (self.thunk, s)
1005 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
1006
1007 def ProcessGroupId(self):
1008 # type: () -> int
1009 """Returns the group ID of this process."""
1010 # This should only ever be called AFTER the process has started
1011 assert self.pid != -1
1012 if self.parent_pipeline:
1013 # XXX: Maybe we should die here instead? Unclear if this branch
1014 # should even be reachable with the current builtins.
1015 return self.parent_pipeline.ProcessGroupId()
1016
1017 return self.pid
1018
1019 def DisplayJob(self, job_id, f, style):
1020 # type: (int, mylib.Writer, int) -> None
1021 if job_id == -1:
1022 job_id_str = ' '
1023 else:
1024 job_id_str = '%%%d' % job_id
1025 if style == STYLE_PID_ONLY:
1026 f.write('%d\n' % self.pid)
1027 else:
1028 f.write('%s %d %7s ' %
1029 (job_id_str, self.pid, _JobStateStr(self.state)))
1030 f.write(self.thunk.UserString())
1031 f.write('\n')
1032
1033 def AddStateChange(self, s):
1034 # type: (ChildStateChange) -> None
1035 self.state_changes.append(s)
1036
1037 def AddPipeToClose(self, r, w):
1038 # type: (int, int) -> None
1039 self.close_r = r
1040 self.close_w = w
1041
1042 def MaybeClosePipe(self):
1043 # type: () -> None
1044 if self.close_r != -1:
1045 posix.close(self.close_r)
1046 posix.close(self.close_w)
1047
1048 def StartProcess(self, why):
1049 # type: (trace_t) -> int
1050 """Start this process with fork(), handling redirects."""
1051 pid = posix.fork()
1052 if pid < 0:
1053 # When does this happen?
1054 e_die('Fatal error in posix.fork()')
1055
1056 elif pid == 0: # child
1057 # Note: this happens in BOTH interactive and non-interactive shells.
1058 # We technically don't need to do most of it in non-interactive, since we
1059 # did not change state in InitInteractiveShell().
1060
1061 for st in self.state_changes:
1062 st.Apply()
1063
1064 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1065 # shouldn't have this.
1066 # https://docs.python.org/2/library/signal.html
1067 # See Python/pythonrun.c.
1068 pyos.Sigaction(SIGPIPE, SIG_DFL)
1069
1070 # Respond to Ctrl-\ (core dump)
1071 pyos.Sigaction(SIGQUIT, SIG_DFL)
1072
1073 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1074 # foreground because suspending them is difficult with our 'lastpipe'
1075 # semantics.
1076 pid = posix.getpid()
1077 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1078 pyos.Sigaction(SIGTSTP, SIG_DFL)
1079
1080 # More signals from
1081 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1082 # (but not SIGCHLD)
1083 pyos.Sigaction(SIGTTOU, SIG_DFL)
1084 pyos.Sigaction(SIGTTIN, SIG_DFL)
1085
1086 self.tracer.OnNewProcess(pid)
1087 # clear foreground pipeline for subshells
1088 self.thunk.Run()
1089 # Never returns
1090
1091 #log('STARTED process %s, pid = %d', self, pid)
1092 self.tracer.OnProcessStart(pid, why)
1093
1094 # Class invariant: after the process is started, it stores its PID.
1095 self.pid = pid
1096
1097 # SetPgid needs to be applied from the child and the parent to avoid
1098 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1099 for st in self.state_changes:
1100 st.ApplyFromParent(self)
1101
1102 # Program invariant: We keep track of every child process!
1103 self.job_list.AddChildProcess(pid, self)
1104
1105 return pid
1106
1107 def Wait(self, waiter):
1108 # type: (Waiter) -> int
1109 """Wait for this process to finish."""
1110 while self.state == job_state_e.Running:
1111 # Only return if there's nothing to wait for. Keep waiting if we were
1112 # interrupted with a signal.
1113 if waiter.WaitForOne() == W1_ECHILD:
1114 break
1115
1116 assert self.status >= 0, self.status
1117 return self.status
1118
1119 def JobWait(self, waiter):
1120 # type: (Waiter) -> wait_status_t
1121 # wait builtin can be interrupted
1122 while self.state == job_state_e.Running:
1123 result = waiter.WaitForOne()
1124
1125 if result >= 0: # signal
1126 return wait_status.Cancelled(result)
1127
1128 if result == W1_ECHILD:
1129 break
1130
1131 return wait_status.Proc(self.status)
1132
1133 def WhenStopped(self, stop_sig):
1134 # type: (int) -> None
1135
1136 # 128 is a shell thing
1137 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1138 self.status = 128 + stop_sig
1139 self.state = job_state_e.Stopped
1140
1141 if self.job_id == -1:
1142 # This process was started in the foreground
1143 self.job_list.AddJob(self)
1144
1145 if not self.in_background:
1146 self.job_control.MaybeTakeTerminal()
1147 self.SetBackground()
1148
1149 def WhenDone(self, pid, status):
1150 # type: (int, int) -> None
1151 """Called by the Waiter when this Process finishes."""
1152
1153 #log('Process WhenDone %d %d', pid, status)
1154 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1155 self.status = status
1156 self.state = job_state_e.Done
1157 if self.parent_pipeline:
1158 self.parent_pipeline.WhenDone(pid, status)
1159 else:
1160 if self.job_id != -1:
1161 # Job might have been brought to the foreground after being
1162 # assigned a job ID.
1163 if self.in_background:
1164 # TODO: bash only prints this interactively
1165 print_stderr('[%d] Done PID %d' % (self.job_id, self.pid))
1166
1167 self.job_list.RemoveJob(self.job_id)
1168
1169 self.job_list.RemoveChildProcess(self.pid)
1170
1171 if not self.in_background:
1172 self.job_control.MaybeTakeTerminal()
1173
1174 def RunProcess(self, waiter, why):
1175 # type: (Waiter, trace_t) -> int
1176 """Run this process synchronously."""
1177 self.StartProcess(why)
1178 # ShellExecutor might be calling this for the last part of a pipeline.
1179 if self.parent_pipeline is None:
1180 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1181 # calling getpgid()?
1182 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1183 return self.Wait(waiter)
1184
1185
1186class ctx_Pipe(object):
1187
1188 def __init__(self, fd_state, fd, err_out):
1189 # type: (FdState, int, List[error.IOError_OSError]) -> None
1190 fd_state.PushStdinFromPipe(fd)
1191 self.fd_state = fd_state
1192 self.err_out = err_out
1193
1194 def __enter__(self):
1195 # type: () -> None
1196 pass
1197
1198 def __exit__(self, type, value, traceback):
1199 # type: (Any, Any, Any) -> None
1200 self.fd_state.Pop(self.err_out)
1201
1202
1203class Pipeline(Job):
1204 """A pipeline of processes to run.
1205
1206 Cases we handle:
1207
1208 foo | bar
1209 $(foo | bar)
1210 foo | bar | read v
1211 """
1212
1213 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1214 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1215 Job.__init__(self)
1216 self.job_control = job_control
1217 self.job_list = job_list
1218 self.tracer = tracer
1219
1220 self.procs = [] # type: List[Process]
1221 self.pids = [] # type: List[int] # pids in order
1222 self.pipe_status = [] # type: List[int] # status in order
1223 self.status = -1 # for 'wait' jobs
1224
1225 self.pgid = INVALID_PGID
1226
1227 # Optional for foreground
1228 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1229 self.last_pipe = None # type: Tuple[int, int]
1230
1231 self.sigpipe_status_ok = sigpipe_status_ok
1232
1233 def ProcessGroupId(self):
1234 # type: () -> int
1235 """Returns the group ID of this pipeline."""
1236 return self.pgid
1237
1238 def DisplayJob(self, job_id, f, style):
1239 # type: (int, mylib.Writer, int) -> None
1240 if style == STYLE_PID_ONLY:
1241 f.write('%d\n' % self.procs[0].pid)
1242 else:
1243 # Note: this is STYLE_LONG.
1244 for i, proc in enumerate(self.procs):
1245 if i == 0: # show job ID for first element in pipeline
1246 job_id_str = '%%%d' % job_id
1247 else:
1248 job_id_str = ' ' # 2 spaces
1249
1250 f.write('%s %d %7s ' %
1251 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1252 f.write(proc.thunk.UserString())
1253 f.write('\n')
1254
1255 def DebugPrint(self):
1256 # type: () -> None
1257 print('Pipeline in state %s' % _JobStateStr(self.state))
1258 if mylib.PYTHON: # %s for Process not allowed in C++
1259 for proc in self.procs:
1260 print(' proc %s' % proc)
1261 _, last_node = self.last_thunk
1262 print(' last %s' % last_node)
1263 print(' pipe_status %s' % self.pipe_status)
1264
1265 def Add(self, p):
1266 # type: (Process) -> None
1267 """Append a process to the pipeline."""
1268 if len(self.procs) == 0:
1269 self.procs.append(p)
1270 return
1271
1272 r, w = posix.pipe()
1273 #log('pipe for %s: %d %d', p, r, w)
1274 prev = self.procs[-1]
1275
1276 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1277 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1278
1279 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1280
1281 self.procs.append(p)
1282
1283 def AddLast(self, thunk):
1284 # type: (Tuple[CommandEvaluator, command_t]) -> None
1285 """Append the last noden to the pipeline.
1286
1287 This is run in the CURRENT process. It is OPTIONAL, because
1288 pipelines in the background are run uniformly.
1289 """
1290 self.last_thunk = thunk
1291
1292 assert len(self.procs) != 0
1293
1294 r, w = posix.pipe()
1295 prev = self.procs[-1]
1296 prev.AddStateChange(StdoutToPipe(r, w))
1297
1298 self.last_pipe = (r, w) # So we can connect it to last_thunk
1299
1300 def StartPipeline(self, waiter):
1301 # type: (Waiter) -> None
1302
1303 # If we are creating a pipeline in a subshell or we aren't running with job
1304 # control, our children should remain in our inherited process group.
1305 # the pipelines's group ID.
1306 if self.job_control.Enabled():
1307 self.pgid = OWN_LEADER # first process in pipeline is the leader
1308
1309 for i, proc in enumerate(self.procs):
1310 if self.pgid != INVALID_PGID:
1311 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1312
1313 # Figure out the pid
1314 pid = proc.StartProcess(trace.PipelinePart)
1315 if i == 0 and self.pgid != INVALID_PGID:
1316 # Mimic bash and use the PID of the FIRST process as the group for the
1317 # whole pipeline.
1318 self.pgid = pid
1319
1320 self.pids.append(pid)
1321 self.pipe_status.append(-1) # uninitialized
1322
1323 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1324 # It can't be done at the end; otherwise processes will have descriptors
1325 # from non-adjacent pipes.
1326 proc.MaybeClosePipe()
1327
1328 if self.last_thunk:
1329 self.pipe_status.append(-1) # for self.last_thunk
1330
1331 def LastPid(self):
1332 # type: () -> int
1333 """For the odd $! variable.
1334
1335 It would be better if job IDs or PGIDs were used consistently.
1336 """
1337 return self.pids[-1]
1338
1339 def Wait(self, waiter):
1340 # type: (Waiter) -> List[int]
1341 """Wait for this pipeline to finish."""
1342
1343 assert self.procs, "no procs for Wait()"
1344 # waitpid(-1) zero or more times
1345 while self.state == job_state_e.Running:
1346 # Keep waiting until there's nothing to wait for.
1347 if waiter.WaitForOne() == W1_ECHILD:
1348 break
1349
1350 return self.pipe_status
1351
1352 def JobWait(self, waiter):
1353 # type: (Waiter) -> wait_status_t
1354 """Called by 'wait' builtin, e.g. 'wait %1'."""
1355 # wait builtin can be interrupted
1356 assert self.procs, "no procs for Wait()"
1357 while self.state == job_state_e.Running:
1358 result = waiter.WaitForOne()
1359
1360 if result >= 0: # signal
1361 return wait_status.Cancelled(result)
1362
1363 if result == W1_ECHILD:
1364 break
1365
1366 return wait_status.Pipeline(self.pipe_status)
1367
1368 def RunLastPart(self, waiter, fd_state):
1369 # type: (Waiter, FdState) -> List[int]
1370 """Run this pipeline synchronously (foreground pipeline).
1371
1372 Returns:
1373 pipe_status (list of integers).
1374 """
1375 assert len(self.pids) == len(self.procs)
1376
1377 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1378 from osh import cmd_eval
1379
1380 # This is tcsetpgrp()
1381 # TODO: fix race condition -- I believe the first process could have
1382 # stopped already, and thus getpgid() will fail
1383 self.job_control.MaybeGiveTerminal(self.pgid)
1384
1385 # Run the last part of the pipeline IN PARALLEL with other processes. It
1386 # may or may not fork:
1387 # echo foo | read line # no fork, the builtin runs in THIS shell process
1388 # ls | wc -l # fork for 'wc'
1389
1390 cmd_ev, last_node = self.last_thunk
1391
1392 assert self.last_pipe is not None
1393 r, w = self.last_pipe # set in AddLast()
1394 posix.close(w) # we will not write here
1395
1396 # Fix lastpipe / job control / DEBUG trap interaction
1397 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1398
1399 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1400 # a pipeline.
1401 cmd_flags |= cmd_eval.NoErrTrap
1402
1403 io_errors = [] # type: List[error.IOError_OSError]
1404 with ctx_Pipe(fd_state, r, io_errors):
1405 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1406
1407 if len(io_errors):
1408 e_die('Error setting up last part of pipeline: %s' %
1409 pyutil.strerror(io_errors[0]))
1410
1411 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1412 # /dev/urandom | sleep 1' will never get SIGPIPE.
1413 posix.close(r)
1414
1415 self.pipe_status[-1] = cmd_ev.LastStatus()
1416 if self.AllDone():
1417 self.state = job_state_e.Done
1418
1419 #log('pipestatus before all have finished = %s', self.pipe_status)
1420 return self.Wait(waiter)
1421
1422 def AllDone(self):
1423 # type: () -> bool
1424
1425 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1426 for status in self.pipe_status:
1427 if status == -1:
1428 return False
1429 return True
1430
1431 def WhenDone(self, pid, status):
1432 # type: (int, int) -> None
1433 """Called by Process.WhenDone."""
1434 #log('Pipeline WhenDone %d %d', pid, status)
1435 i = self.pids.index(pid)
1436 assert i != -1, 'Unexpected PID %d' % pid
1437
1438 if status == 141 and self.sigpipe_status_ok:
1439 status = 0
1440
1441 self.job_list.RemoveChildProcess(pid)
1442 self.pipe_status[i] = status
1443 if self.AllDone():
1444 if self.job_id != -1:
1445 # Job might have been brought to the foreground after being
1446 # assigned a job ID.
1447 if self.in_background:
1448 print_stderr('[%d] Done PGID %d' %
1449 (self.job_id, self.pids[0]))
1450
1451 self.job_list.RemoveJob(self.job_id)
1452
1453 # status of pipeline is status of last process
1454 self.status = self.pipe_status[-1]
1455 self.state = job_state_e.Done
1456 if not self.in_background:
1457 self.job_control.MaybeTakeTerminal()
1458
1459
1460def _JobStateStr(i):
1461 # type: (job_state_t) -> str
1462 return job_state_str(i)[10:] # remove 'job_state.'
1463
1464
1465def _GetTtyFd():
1466 # type: () -> int
1467 """Returns -1 if stdio is not a TTY."""
1468 try:
1469 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1470 except (IOError, OSError) as e:
1471 return -1
1472
1473
1474class ctx_TerminalControl(object):
1475
1476 def __init__(self, job_control, errfmt):
1477 # type: (JobControl, ui.ErrorFormatter) -> None
1478 job_control.InitJobControl()
1479 self.job_control = job_control
1480 self.errfmt = errfmt
1481
1482 def __enter__(self):
1483 # type: () -> None
1484 pass
1485
1486 def __exit__(self, type, value, traceback):
1487 # type: (Any, Any, Any) -> None
1488
1489 # Return the TTY to the original owner before exiting.
1490 try:
1491 self.job_control.MaybeReturnTerminal()
1492 except error.FatalRuntime as e:
1493 # Don't abort the shell on error, just print a message.
1494 self.errfmt.PrettyPrintError(e)
1495
1496
1497class JobControl(object):
1498 """Interface to setpgid(), tcsetpgrp(), etc."""
1499
1500 def __init__(self):
1501 # type: () -> None
1502
1503 # The main shell's PID and group ID.
1504 self.shell_pid = -1
1505 self.shell_pgid = -1
1506
1507 # The fd of the controlling tty. Set to -1 when job control is disabled.
1508 self.shell_tty_fd = -1
1509
1510 # For giving the terminal back to our parent before exiting (if not a login
1511 # shell).
1512 self.original_tty_pgid = -1
1513
1514 def InitJobControl(self):
1515 # type: () -> None
1516 self.shell_pid = posix.getpid()
1517 orig_shell_pgid = posix.getpgid(0)
1518 self.shell_pgid = orig_shell_pgid
1519 self.shell_tty_fd = _GetTtyFd()
1520
1521 # If we aren't the leader of our process group, create a group and mark
1522 # ourselves as the leader.
1523 if self.shell_pgid != self.shell_pid:
1524 try:
1525 posix.setpgid(self.shell_pid, self.shell_pid)
1526 self.shell_pgid = self.shell_pid
1527 except (IOError, OSError) as e:
1528 self.shell_tty_fd = -1
1529
1530 if self.shell_tty_fd != -1:
1531 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1532
1533 # If stdio is a TTY, put the shell's process group in the foreground.
1534 try:
1535 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1536 except (IOError, OSError) as e:
1537 # We probably aren't in the session leader's process group. Disable job
1538 # control.
1539 self.shell_tty_fd = -1
1540 self.shell_pgid = orig_shell_pgid
1541 posix.setpgid(self.shell_pid, self.shell_pgid)
1542
1543 def Enabled(self):
1544 # type: () -> bool
1545
1546 # TODO: get rid of this syscall? SubProgramThunk should set a flag I
1547 # think.
1548 curr_pid = posix.getpid()
1549 # Only the main shell should bother with job control functions.
1550 return curr_pid == self.shell_pid and self.shell_tty_fd != -1
1551
1552 # TODO: This isn't a PID. This is a process group ID?
1553 #
1554 # What should the table look like?
1555 #
1556 # Do we need the last PID? I don't know why bash prints that. Probably so
1557 # you can do wait $!
1558 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1559 #
1560 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1561 # job_id is just an integer. This is sort of lame.
1562 #
1563 # [job_id, flag, pgid, job_state, node]
1564
1565 def MaybeGiveTerminal(self, pgid):
1566 # type: (int) -> None
1567 """If stdio is a TTY, move the given process group to the
1568 foreground."""
1569 if not self.Enabled():
1570 # Only call tcsetpgrp when job control is enabled.
1571 return
1572
1573 try:
1574 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1575 except (IOError, OSError) as e:
1576 e_die('osh: Failed to move process group %d to foreground: %s' %
1577 (pgid, pyutil.strerror(e)))
1578
1579 def MaybeTakeTerminal(self):
1580 # type: () -> None
1581 """If stdio is a TTY, return the main shell's process group to the
1582 foreground."""
1583 self.MaybeGiveTerminal(self.shell_pgid)
1584
1585 def MaybeReturnTerminal(self):
1586 # type: () -> None
1587 """Called before the shell exits."""
1588 self.MaybeGiveTerminal(self.original_tty_pgid)
1589
1590
1591class JobList(object):
1592 """Global list of jobs, used by a few builtins."""
1593
1594 def __init__(self):
1595 # type: () -> None
1596
1597 # job_id -> Job instance
1598 self.jobs = {} # type: Dict[int, Job]
1599
1600 # pid -> Process. This is for STOP notification.
1601 self.child_procs = {} # type: Dict[int, Process]
1602 self.debug_pipelines = [] # type: List[Pipeline]
1603
1604 # Counter used to assign IDs to jobs. It is incremented every time a job
1605 # is created. Once all active jobs are done it is reset to 1. I'm not
1606 # sure if this reset behavior is mandated by POSIX, but other shells do
1607 # it, so we mimic for the sake of compatibility.
1608 self.job_id = 1
1609
1610 def AddJob(self, job):
1611 # type: (Job) -> int
1612 """Add a background job to the list.
1613
1614 A job is either a Process or Pipeline. You can resume a job with 'fg',
1615 kill it with 'kill', etc.
1616
1617 Two cases:
1618
1619 1. async jobs: sleep 5 | sleep 4 &
1620 2. stopped jobs: sleep 5; then Ctrl-Z
1621 """
1622 job_id = self.job_id
1623 self.jobs[job_id] = job
1624 job.job_id = job_id
1625 self.job_id += 1
1626 return job_id
1627
1628 def RemoveJob(self, job_id):
1629 # type: (int) -> None
1630 """Process and Pipeline can call this."""
1631 mylib.dict_erase(self.jobs, job_id)
1632
1633 if len(self.jobs) == 0:
1634 self.job_id = 1
1635
1636 def AddChildProcess(self, pid, proc):
1637 # type: (int, Process) -> None
1638 """Every child process should be added here as soon as we know its PID.
1639
1640 When the Waiter gets an EXITED or STOPPED notification, we need
1641 to know about it so 'jobs' can work.
1642 """
1643 self.child_procs[pid] = proc
1644
1645 def RemoveChildProcess(self, pid):
1646 # type: (int) -> None
1647 """Remove the child process with the given PID."""
1648 mylib.dict_erase(self.child_procs, pid)
1649
1650 if mylib.PYTHON:
1651
1652 def AddPipeline(self, pi):
1653 # type: (Pipeline) -> None
1654 """For debugging only."""
1655 self.debug_pipelines.append(pi)
1656
1657 def ProcessFromPid(self, pid):
1658 # type: (int) -> Process
1659 """For wait $PID.
1660
1661 There's no way to wait for a pipeline with a PID. That uses job
1662 syntax, e.g. %1. Not a great interface.
1663 """
1664 return self.child_procs.get(pid)
1665
1666 def GetCurrentAndPreviousJobs(self):
1667 # type: () -> Tuple[Optional[Job], Optional[Job]]
1668 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1669
1670 See the POSIX specification for the `jobs` builtin for details:
1671 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1672
1673 IMPORTANT NOTE: This method assumes that the jobs list will not change
1674 during its execution! This assumption holds for now because we only ever
1675 update the jobs list from the main loop after WaitPid() informs us of a
1676 change. If we implement `set -b` and install a signal handler for
1677 SIGCHLD we should be careful to synchronize it with this function. The
1678 unsafety of mutating GC data structures from a signal handler should
1679 make this a non-issue, but if bugs related to this appear this note may
1680 be helpful...
1681 """
1682 # Split all active jobs by state and sort each group by decreasing job
1683 # ID to approximate newness.
1684 stopped_jobs = [] # type: List[Job]
1685 running_jobs = [] # type: List[Job]
1686 for i in xrange(0, self.job_id):
1687 job = self.jobs.get(i, None)
1688 if not job:
1689 continue
1690
1691 if job.state == job_state_e.Stopped:
1692 stopped_jobs.append(job)
1693
1694 elif job.state == job_state_e.Running:
1695 running_jobs.append(job)
1696
1697 current = None # type: Optional[Job]
1698 previous = None # type: Optional[Job]
1699 # POSIX says: If there is any suspended job, then the current job shall
1700 # be a suspended job. If there are at least two suspended jobs, then the
1701 # previous job also shall be a suspended job.
1702 #
1703 # So, we will only return running jobs from here if there are no recent
1704 # stopped jobs.
1705 if len(stopped_jobs) > 0:
1706 current = stopped_jobs.pop()
1707
1708 if len(stopped_jobs) > 0:
1709 previous = stopped_jobs.pop()
1710
1711 if len(running_jobs) > 0 and not current:
1712 current = running_jobs.pop()
1713
1714 if len(running_jobs) > 0 and not previous:
1715 previous = running_jobs.pop()
1716
1717 if not previous:
1718 previous = current
1719
1720 return current, previous
1721
1722 def GetJobWithSpec(self, job_spec):
1723 # type: (str) -> Optional[Job]
1724 """Parse the given job spec and return the matching job. If there is no
1725 matching job, this function returns None.
1726
1727 See the POSIX spec for the `jobs` builtin for details about job specs:
1728 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1729 """
1730 if job_spec in CURRENT_JOB_SPECS:
1731 current, _ = self.GetCurrentAndPreviousJobs()
1732 return current
1733
1734 if job_spec == '%-':
1735 _, previous = self.GetCurrentAndPreviousJobs()
1736 return previous
1737
1738 # TODO: Add support for job specs based on prefixes of process argv.
1739 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1740 if m is not None:
1741 assert len(m) == 2
1742 job_id = int(m[1])
1743 if job_id in self.jobs:
1744 return self.jobs[job_id]
1745
1746 return None
1747
1748 def DisplayJobs(self, style):
1749 # type: (int) -> None
1750 """Used by the 'jobs' builtin.
1751
1752 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1753
1754 "By default, the jobs utility shall display the status of all stopped jobs,
1755 running background jobs and all jobs whose status has changed and have not
1756 been reported by the shell."
1757 """
1758 # NOTE: A job is a background process or pipeline.
1759 #
1760 # echo hi | wc -l -- this starts two processes. Wait for TWO
1761 # echo hi | wc -l & -- this starts a process which starts two processes
1762 # Wait for ONE.
1763 #
1764 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1765 # for %% and %-, PID, status, and "command".
1766 #
1767 # Every component of a pipeline is on the same line with 'jobs', but
1768 # they're separated into different lines with 'jobs -l'.
1769 #
1770 # See demo/jobs-builtin.sh
1771
1772 # $ jobs -l
1773 # [1]+ 24414 Stopped sleep 5
1774 # 24415 | sleep 5
1775 # [2] 24502 Running sleep 6
1776 # 24503 | sleep 6
1777 # 24504 | sleep 5 &
1778 # [3]- 24508 Running sleep 6
1779 # 24509 | sleep 6
1780 # 24510 | sleep 5 &
1781
1782 f = mylib.Stdout()
1783 for job_id, job in iteritems(self.jobs):
1784 # Use the %1 syntax
1785 job.DisplayJob(job_id, f, style)
1786
1787 def DebugPrint(self):
1788 # type: () -> None
1789
1790 f = mylib.Stdout()
1791 f.write('\n')
1792 f.write('[process debug info]\n')
1793
1794 for pid, proc in iteritems(self.child_procs):
1795 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1796 #p = ' |' if proc.parent_pipeline else ''
1797 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1798
1799 if len(self.debug_pipelines):
1800 f.write('\n')
1801 f.write('[pipeline debug info]\n')
1802 for pi in self.debug_pipelines:
1803 pi.DebugPrint()
1804
1805 def ListRecent(self):
1806 # type: () -> None
1807 """For jobs -n, which I think is also used in the interactive
1808 prompt."""
1809 pass
1810
1811 def NumRunning(self):
1812 # type: () -> int
1813 """Return the number of running jobs.
1814
1815 Used by 'wait' and 'wait -n'.
1816 """
1817 count = 0
1818 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1819 if job.State() == job_state_e.Running:
1820 count += 1
1821 return count
1822
1823
1824# Some WaitForOne() return values
1825W1_OK = -2 # waitpid(-1) returned
1826W1_ECHILD = -3 # no processes to wait for
1827W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1828
1829
1830class Waiter(object):
1831 """A capability to wait for processes.
1832
1833 This must be a singleton (and is because CommandEvaluator is a singleton).
1834
1835 Invariants:
1836 - Every child process is registered once
1837 - Every child process is waited for
1838
1839 Canonical example of why we need a GLOBAL waiter:
1840
1841 { sleep 3; echo 'done 3'; } &
1842 { sleep 4; echo 'done 4'; } &
1843
1844 # ... do arbitrary stuff ...
1845
1846 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1847
1848 Now when you do wait() after starting the pipeline, you might get a pipeline
1849 process OR a background process! So you have to distinguish between them.
1850 """
1851
1852 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1853 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1854 self.job_list = job_list
1855 self.exec_opts = exec_opts
1856 self.signal_safe = signal_safe
1857 self.tracer = tracer
1858 self.last_status = 127 # wait -n error code
1859
1860 def WaitForOne(self, waitpid_options=0):
1861 # type: (int) -> int
1862 """Wait until the next process returns (or maybe Ctrl-C).
1863
1864 Returns:
1865 One of these negative numbers:
1866 W1_ECHILD Nothing to wait for
1867 W1_OK Caller should keep waiting
1868 UNTRAPPED_SIGWINCH
1869 Or
1870 result > 0 Signal that waitpid() was interrupted with
1871
1872 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1873 will try again.
1874
1875 Callers:
1876 wait -n -- loop until there is one fewer process (TODO)
1877 wait -- loop until there are no processes
1878 wait $! -- loop until job state is Done (process or pipeline)
1879 Process::Wait() -- loop until Process state is done
1880 Pipeline::Wait() -- loop until Pipeline state is done
1881
1882 Comparisons:
1883 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1884 the wait builtin
1885
1886 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1887 loop while (gotsigchld), but that might be a hack for System V!
1888
1889 Should we have a cleaner API like named posix::wait_for_one() ?
1890
1891 wait_result =
1892 ECHILD -- nothing to wait for
1893 | Done(int pid, int status) -- process done
1894 | EINTR(bool sigint) -- may or may not retry
1895 """
1896 pid, status = pyos.WaitPid(waitpid_options)
1897 if pid == 0: # WNOHANG passed, and no state changes
1898 return W1_AGAIN
1899 elif pid < 0: # error case
1900 err_num = status
1901 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1902 if err_num == ECHILD:
1903 return W1_ECHILD # nothing to wait for caller should stop
1904 elif err_num == EINTR: # Bug #858 fix
1905 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1906 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1907 else:
1908 # The signature of waitpid() means this shouldn't happen
1909 raise AssertionError()
1910
1911 # All child processes are supposed to be in this dict. But this may
1912 # legitimately happen if a grandchild outlives the child (its parent).
1913 # Then it is reparented under this process, so we might receive
1914 # notification of its exit, even though we didn't start it. We can't have
1915 # any knowledge of such processes, so print a warning.
1916 if pid not in self.job_list.child_procs:
1917 print_stderr("osh: PID %d stopped, but osh didn't start it" % pid)
1918 return W1_OK
1919
1920 proc = self.job_list.child_procs[pid]
1921 if 0:
1922 self.job_list.DebugPrint()
1923
1924 if WIFSIGNALED(status):
1925 term_sig = WTERMSIG(status)
1926 status = 128 + term_sig
1927
1928 # Print newline after Ctrl-C.
1929 if term_sig == SIGINT:
1930 print('')
1931
1932 proc.WhenDone(pid, status)
1933
1934 elif WIFEXITED(status):
1935 status = WEXITSTATUS(status)
1936 #log('exit status: %s', status)
1937 proc.WhenDone(pid, status)
1938
1939 elif WIFSTOPPED(status):
1940 #status = WEXITSTATUS(status)
1941 stop_sig = WSTOPSIG(status)
1942
1943 print_stderr('')
1944 print_stderr('[PID %d] Stopped with signal %d' % (pid, stop_sig))
1945 proc.WhenStopped(stop_sig)
1946
1947 else:
1948 raise AssertionError(status)
1949
1950 self.last_status = status # for wait -n
1951 self.tracer.OnProcessEnd(pid, status)
1952 return W1_OK
1953
1954 def PollNotifications(self):
1955 # type: () -> None
1956 """
1957 Process all pending state changes.
1958 """
1959 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1960 continue