OILS / core / process.py View on Github | oilshell.org

1964 lines, 951 significant
1# Copyright 2016 Andy Chu. All rights reserved.
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7"""
8process.py - Launch processes and manipulate file descriptors.
9"""
10from __future__ import print_function
11
12from errno import EACCES, EBADF, ECHILD, EINTR, ENOENT, ENOEXEC, EEXIST
13import fcntl as fcntl_
14from fcntl import F_DUPFD, F_GETFD, F_SETFD, FD_CLOEXEC
15from signal import (SIG_DFL, SIG_IGN, SIGINT, SIGPIPE, SIGQUIT, SIGTSTP,
16 SIGTTOU, SIGTTIN, SIGWINCH)
17
18from _devbuild.gen.id_kind_asdl import Id
19from _devbuild.gen.runtime_asdl import (job_state_e, job_state_t,
20 job_state_str, wait_status,
21 wait_status_t, RedirValue,
22 redirect_arg, redirect_arg_e, trace,
23 trace_t)
24from _devbuild.gen.syntax_asdl import (
25 loc_t,
26 redir_loc,
27 redir_loc_e,
28 redir_loc_t,
29)
30from _devbuild.gen.value_asdl import (value, value_e)
31from core import dev
32from core import error
33from core.error import e_die
34from core import pyutil
35from core import pyos
36from core import state
37from display import ui
38from core import util
39from data_lang import j8_lite
40from frontend import location
41from frontend import match
42from mycpp import mylib
43from mycpp.mylib import log, print_stderr, probe, tagswitch, iteritems
44
45import posix_ as posix
46from posix_ import (
47 # translated by mycpp and directly called! No wrapper!
48 WIFSIGNALED,
49 WIFEXITED,
50 WIFSTOPPED,
51 WEXITSTATUS,
52 WSTOPSIG,
53 WTERMSIG,
54 WNOHANG,
55 O_APPEND,
56 O_CREAT,
57 O_EXCL,
58 O_NONBLOCK,
59 O_NOCTTY,
60 O_RDONLY,
61 O_RDWR,
62 O_WRONLY,
63 O_TRUNC,
64)
65
66from typing import IO, List, Tuple, Dict, Optional, Any, cast, TYPE_CHECKING
67
68if TYPE_CHECKING:
69 from _devbuild.gen.runtime_asdl import cmd_value
70 from _devbuild.gen.syntax_asdl import command_t
71 from builtin import trap_osh
72 from core import optview
73 from core.util import _DebugFile
74 from osh.cmd_eval import CommandEvaluator
75
76NO_FD = -1
77
78# Minimum file descriptor that the shell can use. Other descriptors can be
79# directly used by user programs, e.g. exec 9>&1
80#
81# Oil uses 100 because users are allowed TWO digits in frontend/lexer_def.py.
82# This is a compromise between bash (unlimited, but requires crazy
83# bookkeeping), and dash/zsh (10) and mksh (24)
84_SHELL_MIN_FD = 100
85
86# Style for 'jobs' builtin
87STYLE_DEFAULT = 0
88STYLE_LONG = 1
89STYLE_PID_ONLY = 2
90
91# To save on allocations in JobList::GetJobWithSpec()
92CURRENT_JOB_SPECS = ['', '%', '%%', '%+']
93
94
95class ctx_FileCloser(object):
96
97 def __init__(self, f):
98 # type: (mylib.LineReader) -> None
99 self.f = f
100
101 def __enter__(self):
102 # type: () -> None
103 pass
104
105 def __exit__(self, type, value, traceback):
106 # type: (Any, Any, Any) -> None
107 self.f.close()
108
109
110def InitInteractiveShell():
111 # type: () -> None
112 """Called when initializing an interactive shell."""
113
114 # The shell itself should ignore Ctrl-\.
115 pyos.Sigaction(SIGQUIT, SIG_IGN)
116
117 # This prevents Ctrl-Z from suspending OSH in interactive mode.
118 pyos.Sigaction(SIGTSTP, SIG_IGN)
119
120 # More signals from
121 # https://www.gnu.org/software/libc/manual/html_node/Initializing-the-Shell.html
122 # (but not SIGCHLD)
123 pyos.Sigaction(SIGTTOU, SIG_IGN)
124 pyos.Sigaction(SIGTTIN, SIG_IGN)
125
126 # Register a callback to receive terminal width changes.
127 # NOTE: In line_input.c, we turned off rl_catch_sigwinch.
128
129 # This is ALWAYS on, which means that it can cause EINTR, and wait() and
130 # read() have to handle it
131 pyos.RegisterSignalInterest(SIGWINCH)
132
133
134def SaveFd(fd):
135 # type: (int) -> int
136 saved = fcntl_.fcntl(fd, F_DUPFD, _SHELL_MIN_FD) # type: int
137 return saved
138
139
140class _RedirFrame(object):
141
142 def __init__(self, saved_fd, orig_fd, forget):
143 # type: (int, int, bool) -> None
144 self.saved_fd = saved_fd
145 self.orig_fd = orig_fd
146 self.forget = forget
147
148
149class _FdFrame(object):
150
151 def __init__(self):
152 # type: () -> None
153 self.saved = [] # type: List[_RedirFrame]
154 self.need_wait = [] # type: List[Process]
155
156 def Forget(self):
157 # type: () -> None
158 """For exec 1>&2."""
159 for rf in reversed(self.saved):
160 if rf.saved_fd != NO_FD and rf.forget:
161 posix.close(rf.saved_fd)
162
163 del self.saved[:] # like list.clear() in Python 3.3
164 del self.need_wait[:]
165
166 def __repr__(self):
167 # type: () -> str
168 return '<_FdFrame %s>' % self.saved
169
170
171class FdState(object):
172 """File descriptor state for the current process.
173
174 For example, you can do 'myfunc > out.txt' without forking. Child
175 processes inherit our state.
176 """
177
178 def __init__(
179 self,
180 errfmt, # type: ui.ErrorFormatter
181 job_control, # type: JobControl
182 job_list, # type: JobList
183 mem, # type: state.Mem
184 tracer, # type: Optional[dev.Tracer]
185 waiter, # type: Optional[Waiter]
186 exec_opts, # type: optview.Exec
187 ):
188 # type: (...) -> None
189 """
190 Args:
191 errfmt: for errors
192 job_list: For keeping track of _HereDocWriterThunk
193 """
194 self.errfmt = errfmt
195 self.job_control = job_control
196 self.job_list = job_list
197 self.cur_frame = _FdFrame() # for the top level
198 self.stack = [self.cur_frame]
199 self.mem = mem
200 self.tracer = tracer
201 self.waiter = waiter
202 self.exec_opts = exec_opts
203
204 def Open(self, path):
205 # type: (str) -> mylib.LineReader
206 """Opens a path for read, but moves it out of the reserved 3-9 fd
207 range.
208
209 Returns:
210 A Python file object. The caller is responsible for Close().
211
212 Raises:
213 IOError or OSError if the path can't be found. (This is Python-induced wart)
214 """
215 fd_mode = O_RDONLY
216 f = self._Open(path, 'r', fd_mode)
217
218 # Hacky downcast
219 return cast('mylib.LineReader', f)
220
221 # used for util.DebugFile
222 def OpenForWrite(self, path):
223 # type: (str) -> mylib.Writer
224 fd_mode = O_CREAT | O_RDWR
225 f = self._Open(path, 'w', fd_mode)
226
227 # Hacky downcast
228 return cast('mylib.Writer', f)
229
230 def _Open(self, path, c_mode, fd_mode):
231 # type: (str, str, int) -> IO[str]
232 fd = posix.open(path, fd_mode, 0o666) # may raise OSError
233
234 # Immediately move it to a new location
235 new_fd = SaveFd(fd)
236 posix.close(fd)
237
238 # Return a Python file handle
239 f = posix.fdopen(new_fd, c_mode) # may raise IOError
240 return f
241
242 def _WriteFdToMem(self, fd_name, fd):
243 # type: (str, int) -> None
244 if self.mem:
245 # setvar, not setref
246 state.OshLanguageSetValue(self.mem, location.LName(fd_name),
247 value.Str(str(fd)))
248
249 def _ReadFdFromMem(self, fd_name):
250 # type: (str) -> int
251 val = self.mem.GetValue(fd_name)
252 if val.tag() == value_e.Str:
253 try:
254 return int(cast(value.Str, val).s)
255 except ValueError:
256 return NO_FD
257 return NO_FD
258
259 def _PushSave(self, fd):
260 # type: (int) -> bool
261 """Save fd to a new location and remember to restore it later."""
262 #log('---- _PushSave %s', fd)
263 ok = True
264 try:
265 new_fd = SaveFd(fd)
266 except (IOError, OSError) as e:
267 ok = False
268 # Example program that causes this error: exec 4>&1. Descriptor 4 isn't
269 # open.
270 # This seems to be ignored in dash too in savefd()?
271 if e.errno != EBADF:
272 raise
273 if ok:
274 posix.close(fd)
275 fcntl_.fcntl(new_fd, F_SETFD, FD_CLOEXEC)
276 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, True))
277 else:
278 # if we got EBADF, we still need to close the original on Pop()
279 self._PushClose(fd)
280
281 return ok
282
283 def _PushDup(self, fd1, blame_loc):
284 # type: (int, redir_loc_t) -> int
285 """Save fd2 in a higher range, and dup fd1 onto fd2.
286
287 Returns whether F_DUPFD/dup2 succeeded, and the new descriptor.
288 """
289 UP_loc = blame_loc
290 if blame_loc.tag() == redir_loc_e.VarName:
291 fd2_name = cast(redir_loc.VarName, UP_loc).name
292 try:
293 # F_DUPFD: GREATER than range
294 new_fd = fcntl_.fcntl(fd1, F_DUPFD, _SHELL_MIN_FD) # type: int
295 except (IOError, OSError) as e:
296 if e.errno == EBADF:
297 print_stderr('F_DUPFD fd %d: %s' %
298 (fd1, pyutil.strerror(e)))
299 return NO_FD
300 else:
301 raise # this redirect failed
302
303 self._WriteFdToMem(fd2_name, new_fd)
304
305 elif blame_loc.tag() == redir_loc_e.Fd:
306 fd2 = cast(redir_loc.Fd, UP_loc).fd
307
308 if fd1 == fd2:
309 # The user could have asked for it to be open on descriptor 3, but open()
310 # already returned 3, e.g. echo 3>out.txt
311 return NO_FD
312
313 # Check the validity of fd1 before _PushSave(fd2)
314 try:
315 fcntl_.fcntl(fd1, F_GETFD)
316 except (IOError, OSError) as e:
317 print_stderr('F_GETFD fd %d: %s' % (fd1, pyutil.strerror(e)))
318 raise
319
320 need_restore = self._PushSave(fd2)
321
322 #log('==== dup2 %s %s\n' % (fd1, fd2))
323 try:
324 posix.dup2(fd1, fd2)
325 except (IOError, OSError) as e:
326 # bash/dash give this error too, e.g. for 'echo hi 1>&3'
327 print_stderr('dup2(%d, %d): %s' %
328 (fd1, fd2, pyutil.strerror(e)))
329
330 # Restore and return error
331 if need_restore:
332 rf = self.cur_frame.saved.pop()
333 posix.dup2(rf.saved_fd, rf.orig_fd)
334 posix.close(rf.saved_fd)
335
336 raise # this redirect failed
337
338 new_fd = fd2
339
340 else:
341 raise AssertionError()
342
343 return new_fd
344
345 def _PushCloseFd(self, blame_loc):
346 # type: (redir_loc_t) -> bool
347 """For 2>&-"""
348 # exec {fd}>&- means close the named descriptor
349
350 UP_loc = blame_loc
351 if blame_loc.tag() == redir_loc_e.VarName:
352 fd_name = cast(redir_loc.VarName, UP_loc).name
353 fd = self._ReadFdFromMem(fd_name)
354 if fd == NO_FD:
355 return False
356
357 elif blame_loc.tag() == redir_loc_e.Fd:
358 fd = cast(redir_loc.Fd, UP_loc).fd
359
360 else:
361 raise AssertionError()
362
363 self._PushSave(fd)
364
365 return True
366
367 def _PushClose(self, fd):
368 # type: (int) -> None
369 self.cur_frame.saved.append(_RedirFrame(NO_FD, fd, False))
370
371 def _PushWait(self, proc):
372 # type: (Process) -> None
373 self.cur_frame.need_wait.append(proc)
374
375 def _ApplyRedirect(self, r):
376 # type: (RedirValue) -> None
377 arg = r.arg
378 UP_arg = arg
379 with tagswitch(arg) as case:
380
381 if case(redirect_arg_e.Path):
382 arg = cast(redirect_arg.Path, UP_arg)
383 # noclobber flag is OR'd with other flags when allowed
384 noclobber_mode = O_EXCL if self.exec_opts.noclobber() else 0
385 if r.op_id in (Id.Redir_Great, Id.Redir_AndGreat): # > &>
386 # NOTE: This is different than >| because it respects noclobber, but
387 # that option is almost never used. See test/wild.sh.
388 mode = O_CREAT | O_WRONLY | O_TRUNC | noclobber_mode
389 elif r.op_id == Id.Redir_Clobber: # >|
390 mode = O_CREAT | O_WRONLY | O_TRUNC
391 elif r.op_id in (Id.Redir_DGreat,
392 Id.Redir_AndDGreat): # >> &>>
393 mode = O_CREAT | O_WRONLY | O_APPEND | noclobber_mode
394 elif r.op_id == Id.Redir_Less: # <
395 mode = O_RDONLY
396 elif r.op_id == Id.Redir_LessGreat: # <>
397 mode = O_CREAT | O_RDWR
398 else:
399 raise NotImplementedError(r.op_id)
400
401 # NOTE: 0666 is affected by umask, all shells use it.
402 try:
403 open_fd = posix.open(arg.filename, mode, 0o666)
404 except (IOError, OSError) as e:
405 if e.errno == EEXIST and self.exec_opts.noclobber():
406 extra = ' (noclobber)'
407 else:
408 extra = ''
409 self.errfmt.Print_(
410 "Can't open %r: %s%s" %
411 (arg.filename, pyutil.strerror(e), extra),
412 blame_loc=r.op_loc)
413 raise # redirect failed
414
415 new_fd = self._PushDup(open_fd, r.loc)
416 if new_fd != NO_FD:
417 posix.close(open_fd)
418
419 # Now handle &> and &>> and their variants. These pairs are the same:
420 #
421 # stdout_stderr.py &> out-err.txt
422 # stdout_stderr.py > out-err.txt 2>&1
423 #
424 # stdout_stderr.py 3&> out-err.txt
425 # stdout_stderr.py 3> out-err.txt 2>&3
426 #
427 # Ditto for {fd}> and {fd}&>
428
429 if r.op_id in (Id.Redir_AndGreat, Id.Redir_AndDGreat):
430 self._PushDup(new_fd, redir_loc.Fd(2))
431
432 elif case(redirect_arg_e.CopyFd): # e.g. echo hi 1>&2
433 arg = cast(redirect_arg.CopyFd, UP_arg)
434
435 if r.op_id == Id.Redir_GreatAnd: # 1>&2
436 self._PushDup(arg.target_fd, r.loc)
437
438 elif r.op_id == Id.Redir_LessAnd: # 0<&5
439 # The only difference between >& and <& is the default file
440 # descriptor argument.
441 self._PushDup(arg.target_fd, r.loc)
442
443 else:
444 raise NotImplementedError()
445
446 elif case(redirect_arg_e.MoveFd): # e.g. echo hi 5>&6-
447 arg = cast(redirect_arg.MoveFd, UP_arg)
448 new_fd = self._PushDup(arg.target_fd, r.loc)
449 if new_fd != NO_FD:
450 posix.close(arg.target_fd)
451
452 UP_loc = r.loc
453 if r.loc.tag() == redir_loc_e.Fd:
454 fd = cast(redir_loc.Fd, UP_loc).fd
455 else:
456 fd = NO_FD
457
458 self.cur_frame.saved.append(_RedirFrame(new_fd, fd, False))
459
460 elif case(redirect_arg_e.CloseFd): # e.g. echo hi 5>&-
461 self._PushCloseFd(r.loc)
462
463 elif case(redirect_arg_e.HereDoc):
464 arg = cast(redirect_arg.HereDoc, UP_arg)
465
466 # NOTE: Do these descriptors have to be moved out of the range 0-9?
467 read_fd, write_fd = posix.pipe()
468
469 self._PushDup(read_fd, r.loc) # stdin is now the pipe
470
471 # We can't close like we do in the filename case above? The writer can
472 # get a "broken pipe".
473 self._PushClose(read_fd)
474
475 thunk = _HereDocWriterThunk(write_fd, arg.body)
476
477 # Use PIPE_SIZE to save a process in the case of small here
478 # docs, which are the common case. (dash does this.)
479
480 # Note: could instrument this to see how often it happens.
481 # Though strace -ff can also work.
482 start_process = len(arg.body) > 4096
483 #start_process = True
484
485 if start_process:
486 here_proc = Process(thunk, self.job_control, self.job_list,
487 self.tracer)
488
489 # NOTE: we could close the read pipe here, but it doesn't really
490 # matter because we control the code.
491 here_proc.StartProcess(trace.HereDoc)
492 #log('Started %s as %d', here_proc, pid)
493 self._PushWait(here_proc)
494
495 # Now that we've started the child, close it in the parent.
496 posix.close(write_fd)
497
498 else:
499 posix.write(write_fd, arg.body)
500 posix.close(write_fd)
501
502 def Push(self, redirects, err_out):
503 # type: (List[RedirValue], List[error.IOError_OSError]) -> None
504 """Apply a group of redirects and remember to undo them."""
505
506 #log('> fd_state.Push %s', redirects)
507 new_frame = _FdFrame()
508 self.stack.append(new_frame)
509 self.cur_frame = new_frame
510
511 for r in redirects:
512 #log('apply %s', r)
513 with ui.ctx_Location(self.errfmt, r.op_loc):
514 try:
515 self._ApplyRedirect(r)
516 except (IOError, OSError) as e:
517 err_out.append(e)
518 # This can fail too
519 self.Pop(err_out)
520 return # for bad descriptor, etc.
521
522 def PushStdinFromPipe(self, r):
523 # type: (int) -> bool
524 """Save the current stdin and make it come from descriptor 'r'.
525
526 'r' is typically the read-end of a pipe. For 'lastpipe'/ZSH
527 semantics of
528
529 echo foo | read line; echo $line
530 """
531 new_frame = _FdFrame()
532 self.stack.append(new_frame)
533 self.cur_frame = new_frame
534
535 self._PushDup(r, redir_loc.Fd(0))
536 return True
537
538 def Pop(self, err_out):
539 # type: (List[error.IOError_OSError]) -> None
540 frame = self.stack.pop()
541 #log('< Pop %s', frame)
542 for rf in reversed(frame.saved):
543 if rf.saved_fd == NO_FD:
544 #log('Close %d', orig)
545 try:
546 posix.close(rf.orig_fd)
547 except (IOError, OSError) as e:
548 err_out.append(e)
549 log('Error closing descriptor %d: %s', rf.orig_fd,
550 pyutil.strerror(e))
551 return
552 else:
553 try:
554 posix.dup2(rf.saved_fd, rf.orig_fd)
555 except (IOError, OSError) as e:
556 err_out.append(e)
557 log('dup2(%d, %d) error: %s', rf.saved_fd, rf.orig_fd,
558 pyutil.strerror(e))
559 #log('fd state:')
560 #posix.system('ls -l /proc/%s/fd' % posix.getpid())
561 return
562 posix.close(rf.saved_fd)
563 #log('dup2 %s %s', saved, orig)
564
565 # Wait for here doc processes to finish.
566 for proc in frame.need_wait:
567 unused_status = proc.Wait(self.waiter)
568
569 def MakePermanent(self):
570 # type: () -> None
571 self.cur_frame.Forget()
572
573
574class ChildStateChange(object):
575
576 def __init__(self):
577 # type: () -> None
578 """Empty constructor for mycpp."""
579 pass
580
581 def Apply(self):
582 # type: () -> None
583 raise NotImplementedError()
584
585 def ApplyFromParent(self, proc):
586 # type: (Process) -> None
587 """Noop for all state changes other than SetPgid for mycpp."""
588 pass
589
590
591class StdinFromPipe(ChildStateChange):
592
593 def __init__(self, pipe_read_fd, w):
594 # type: (int, int) -> None
595 self.r = pipe_read_fd
596 self.w = w
597
598 def __repr__(self):
599 # type: () -> str
600 return '<StdinFromPipe %d %d>' % (self.r, self.w)
601
602 def Apply(self):
603 # type: () -> None
604 posix.dup2(self.r, 0)
605 posix.close(self.r) # close after dup
606
607 posix.close(self.w) # we're reading from the pipe, not writing
608 #log('child CLOSE w %d pid=%d', self.w, posix.getpid())
609
610
611class StdoutToPipe(ChildStateChange):
612
613 def __init__(self, r, pipe_write_fd):
614 # type: (int, int) -> None
615 self.r = r
616 self.w = pipe_write_fd
617
618 def __repr__(self):
619 # type: () -> str
620 return '<StdoutToPipe %d %d>' % (self.r, self.w)
621
622 def Apply(self):
623 # type: () -> None
624 posix.dup2(self.w, 1)
625 posix.close(self.w) # close after dup
626
627 posix.close(self.r) # we're writing to the pipe, not reading
628 #log('child CLOSE r %d pid=%d', self.r, posix.getpid())
629
630
631INVALID_PGID = -1
632# argument to setpgid() that means the process is its own leader
633OWN_LEADER = 0
634
635
636class SetPgid(ChildStateChange):
637
638 def __init__(self, pgid, tracer):
639 # type: (int, dev.Tracer) -> None
640 self.pgid = pgid
641 self.tracer = tracer
642
643 def Apply(self):
644 # type: () -> None
645 try:
646 posix.setpgid(0, self.pgid)
647 except (IOError, OSError) as e:
648 self.tracer.OtherMessage(
649 'osh: child %d failed to set its process group to %d: %s' %
650 (posix.getpid(), self.pgid, pyutil.strerror(e)))
651
652 def ApplyFromParent(self, proc):
653 # type: (Process) -> None
654 try:
655 posix.setpgid(proc.pid, self.pgid)
656 except (IOError, OSError) as e:
657 self.tracer.OtherMessage(
658 'osh: parent failed to set process group for PID %d to %d: %s'
659 % (proc.pid, self.pgid, pyutil.strerror(e)))
660
661
662class ExternalProgram(object):
663 """The capability to execute an external program like 'ls'."""
664
665 def __init__(
666 self,
667 hijack_shebang, # type: str
668 fd_state, # type: FdState
669 errfmt, # type: ui.ErrorFormatter
670 debug_f, # type: _DebugFile
671 ):
672 # type: (...) -> None
673 """
674 Args:
675 hijack_shebang: The path of an interpreter to run instead of the one
676 specified in the shebang line. May be empty.
677 """
678 self.hijack_shebang = hijack_shebang
679 self.fd_state = fd_state
680 self.errfmt = errfmt
681 self.debug_f = debug_f
682
683 def Exec(self, argv0_path, cmd_val, environ):
684 # type: (str, cmd_value.Argv, Dict[str, str]) -> None
685 """Execute a program and exit this process.
686
687 Called by: ls / exec ls / ( ls / )
688 """
689 probe('process', 'ExternalProgram_Exec', argv0_path)
690 self._Exec(argv0_path, cmd_val.argv, cmd_val.arg_locs[0], environ,
691 True)
692 assert False, "This line should never execute" # NO RETURN
693
694 def _Exec(self, argv0_path, argv, argv0_loc, environ, should_retry):
695 # type: (str, List[str], loc_t, Dict[str, str], bool) -> None
696 if len(self.hijack_shebang):
697 opened = True
698 try:
699 f = self.fd_state.Open(argv0_path)
700 except (IOError, OSError) as e:
701 opened = False
702
703 if opened:
704 with ctx_FileCloser(f):
705 # Test if the shebang looks like a shell. TODO: The file might be
706 # binary with no newlines, so read 80 bytes instead of readline().
707
708 #line = f.read(80) # type: ignore # TODO: fix this
709 line = f.readline()
710
711 if match.ShouldHijack(line):
712 h_argv = [self.hijack_shebang, argv0_path]
713 h_argv.extend(argv[1:])
714 argv = h_argv
715 argv0_path = self.hijack_shebang
716 self.debug_f.writeln('Hijacked: %s' % argv0_path)
717 else:
718 #self.debug_f.log('Not hijacking %s (%r)', argv, line)
719 pass
720
721 try:
722 posix.execve(argv0_path, argv, environ)
723 except (IOError, OSError) as e:
724 # Run with /bin/sh when ENOEXEC error (no shebang). All shells do this.
725 if e.errno == ENOEXEC and should_retry:
726 new_argv = ['/bin/sh', argv0_path]
727 new_argv.extend(argv[1:])
728 self._Exec('/bin/sh', new_argv, argv0_loc, environ, False)
729 # NO RETURN
730
731 # Would be nice: when the path is relative and ENOENT: print PWD and do
732 # spelling correction?
733
734 self.errfmt.Print_(
735 "Can't execute %r: %s" % (argv0_path, pyutil.strerror(e)),
736 argv0_loc)
737
738 # POSIX mentions 126 and 127 for two specific errors. The rest are
739 # unspecified.
740 #
741 # http://pubs.opengroup.org/onlinepubs/9699919799.2016edition/utilities/V3_chap02.html#tag_18_08_02
742 if e.errno == EACCES:
743 status = 126
744 elif e.errno == ENOENT:
745 # TODO: most shells print 'command not found', rather than strerror()
746 # == "No such file or directory". That's better because it's at the
747 # end of the path search, and we're never searching for a directory.
748 status = 127
749 else:
750 # dash uses 2, but we use that for parse errors. This seems to be
751 # consistent with mksh and zsh.
752 status = 127
753
754 posix._exit(status)
755 # NO RETURN
756
757
758class Thunk(object):
759 """Abstract base class for things runnable in another process."""
760
761 def __init__(self):
762 # type: () -> None
763 """Empty constructor for mycpp."""
764 pass
765
766 def Run(self):
767 # type: () -> None
768 """Returns a status code."""
769 raise NotImplementedError()
770
771 def UserString(self):
772 # type: () -> str
773 """Display for the 'jobs' list."""
774 raise NotImplementedError()
775
776 def __repr__(self):
777 # type: () -> str
778 return self.UserString()
779
780
781class ExternalThunk(Thunk):
782 """An external executable."""
783
784 def __init__(self, ext_prog, argv0_path, cmd_val, environ):
785 # type: (ExternalProgram, str, cmd_value.Argv, Dict[str, str]) -> None
786 self.ext_prog = ext_prog
787 self.argv0_path = argv0_path
788 self.cmd_val = cmd_val
789 self.environ = environ
790
791 def UserString(self):
792 # type: () -> str
793
794 # NOTE: This is the format the Tracer uses.
795 # bash displays sleep $n & (code)
796 # but OSH displays sleep 1 & (argv array)
797 # We could switch the former but I'm not sure it's necessary.
798 tmp = [j8_lite.MaybeShellEncode(a) for a in self.cmd_val.argv]
799 return '[process] %s' % ' '.join(tmp)
800
801 def Run(self):
802 # type: () -> None
803 """An ExternalThunk is run in parent for the exec builtin."""
804 self.ext_prog.Exec(self.argv0_path, self.cmd_val, self.environ)
805
806
807class SubProgramThunk(Thunk):
808 """A subprogram that can be executed in another process."""
809
810 def __init__(self, cmd_ev, node, trap_state, multi_trace, inherit_errexit,
811 inherit_errtrace):
812 # type: (CommandEvaluator, command_t, trap_osh.TrapState, dev.MultiTracer, bool, bool) -> None
813 self.cmd_ev = cmd_ev
814 self.node = node
815 self.trap_state = trap_state
816 self.multi_trace = multi_trace
817 self.inherit_errexit = inherit_errexit # for bash errexit compatibility
818 self.inherit_errtrace = inherit_errtrace # for bash errtrace compatibility
819
820 def UserString(self):
821 # type: () -> str
822
823 # NOTE: These can be pieces of a pipeline, so they're arbitrary nodes.
824 # TODO: Extract SPIDS from node to display source? Note that
825 # CompoundStatus also has locations of each pipeline component; see
826 # Executor.RunPipeline()
827 thunk_str = ui.CommandType(self.node)
828 return '[subprog] %s' % thunk_str
829
830 def Run(self):
831 # type: () -> None
832 #self.errfmt.OneLineErrExit() # don't quote code in child processes
833 probe('process', 'SubProgramThunk_Run')
834
835 # TODO: break circular dep. Bit flags could go in ASDL or headers.
836 from osh import cmd_eval
837
838 # signal handlers aren't inherited
839 self.trap_state.ClearForSubProgram(self.inherit_errtrace)
840
841 # NOTE: may NOT return due to exec().
842 if not self.inherit_errexit:
843 self.cmd_ev.mutable_opts.DisableErrExit()
844 try:
845 # optimize to eliminate redundant subshells like ( echo hi ) | wc -l etc.
846 self.cmd_ev.ExecuteAndCatch(
847 self.node,
848 cmd_eval.OptimizeSubshells | cmd_eval.MarkLastCommands)
849 status = self.cmd_ev.LastStatus()
850 # NOTE: We ignore the is_fatal return value. The user should set -o
851 # errexit so failures in subprocesses cause failures in the parent.
852 except util.UserExit as e:
853 status = e.status
854
855 # Handle errors in a subshell. These two cases are repeated from main()
856 # and the core/completion.py hook.
857 except KeyboardInterrupt:
858 print('')
859 status = 130 # 128 + 2
860 except (IOError, OSError) as e:
861 print_stderr('oils I/O error (subprogram): %s' %
862 pyutil.strerror(e))
863 status = 2
864
865 # If ProcessInit() doesn't turn off buffering, this is needed before
866 # _exit()
867 pyos.FlushStdout()
868
869 self.multi_trace.WriteDumps()
870
871 # We do NOT want to raise SystemExit here. Otherwise dev.Tracer::Pop()
872 # gets called in BOTH processes.
873 # The crash dump seems to be unaffected.
874 posix._exit(status)
875
876
877class _HereDocWriterThunk(Thunk):
878 """Write a here doc to one end of a pipe.
879
880 May be be executed in either a child process or the main shell
881 process.
882 """
883
884 def __init__(self, w, body_str):
885 # type: (int, str) -> None
886 self.w = w
887 self.body_str = body_str
888
889 def UserString(self):
890 # type: () -> str
891
892 # You can hit Ctrl-Z and the here doc writer will be suspended! Other
893 # shells don't have this problem because they use temp files! That's a bit
894 # unfortunate.
895 return '[here doc writer]'
896
897 def Run(self):
898 # type: () -> None
899 """do_exit: For small pipelines."""
900 probe('process', 'HereDocWriterThunk_Run')
901 #log('Writing %r', self.body_str)
902 posix.write(self.w, self.body_str)
903 #log('Wrote %r', self.body_str)
904 posix.close(self.w)
905 #log('Closed %d', self.w)
906
907 posix._exit(0)
908
909
910class Job(object):
911 """Interface for both Process and Pipeline.
912
913 They both can be put in the background and waited on.
914
915 Confusing thing about pipelines in the background: They have TOO MANY NAMES.
916
917 sleep 1 | sleep 2 &
918
919 - The LAST PID is what's printed at the prompt. This is $!, a PROCESS ID and
920 not a JOB ID.
921 # https://www.gnu.org/software/bash/manual/html_node/Special-Parameters.html#Special-Parameters
922 - The process group leader (setpgid) is the FIRST PID.
923 - It's also %1 or %+. The last job started.
924 """
925
926 def __init__(self):
927 # type: () -> None
928 # Initial state with & or Ctrl-Z is Running.
929 self.state = job_state_e.Running
930 self.job_id = -1
931 self.in_background = False
932
933 def DisplayJob(self, job_id, f, style):
934 # type: (int, mylib.Writer, int) -> None
935 raise NotImplementedError()
936
937 def State(self):
938 # type: () -> job_state_t
939 return self.state
940
941 def ProcessGroupId(self):
942 # type: () -> int
943 """Return the process group ID associated with this job."""
944 raise NotImplementedError()
945
946 def JobWait(self, waiter):
947 # type: (Waiter) -> wait_status_t
948 """Wait for this process/pipeline to be stopped or finished."""
949 raise NotImplementedError()
950
951 def SetBackground(self):
952 # type: () -> None
953 """Record that this job is running in the background."""
954 self.in_background = True
955
956 def SetForeground(self):
957 # type: () -> None
958 """Record that this job is running in the foreground."""
959 self.in_background = False
960
961
962class Process(Job):
963 """A process to run.
964
965 TODO: Should we make it clear that this is a FOREGROUND process? A
966 background process is wrapped in a "job". It is unevaluated.
967
968 It provides an API to manipulate file descriptor state in parent and child.
969 """
970
971 def __init__(self, thunk, job_control, job_list, tracer):
972 # type: (Thunk, JobControl, JobList, dev.Tracer) -> None
973 """
974 Args:
975 thunk: Thunk instance
976 job_list: for process bookkeeping
977 """
978 Job.__init__(self)
979 assert isinstance(thunk, Thunk), thunk
980 self.thunk = thunk
981 self.job_control = job_control
982 self.job_list = job_list
983 self.tracer = tracer
984
985 # For pipelines
986 self.parent_pipeline = None # type: Pipeline
987 self.state_changes = [] # type: List[ChildStateChange]
988 self.close_r = -1
989 self.close_w = -1
990
991 self.pid = -1
992 self.status = -1
993
994 def Init_ParentPipeline(self, pi):
995 # type: (Pipeline) -> None
996 """For updating PIPESTATUS."""
997 self.parent_pipeline = pi
998
999 def __repr__(self):
1000 # type: () -> str
1001
1002 # note: be wary of infinite mutual recursion
1003 #s = ' %s' % self.parent_pipeline if self.parent_pipeline else ''
1004 #return '<Process %s%s>' % (self.thunk, s)
1005 return '<Process %s %s>' % (_JobStateStr(self.state), self.thunk)
1006
1007 def ProcessGroupId(self):
1008 # type: () -> int
1009 """Returns the group ID of this process."""
1010 # This should only ever be called AFTER the process has started
1011 assert self.pid != -1
1012 if self.parent_pipeline:
1013 # XXX: Maybe we should die here instead? Unclear if this branch
1014 # should even be reachable with the current builtins.
1015 return self.parent_pipeline.ProcessGroupId()
1016
1017 return self.pid
1018
1019 def DisplayJob(self, job_id, f, style):
1020 # type: (int, mylib.Writer, int) -> None
1021 if job_id == -1:
1022 job_id_str = ' '
1023 else:
1024 job_id_str = '%%%d' % job_id
1025 if style == STYLE_PID_ONLY:
1026 f.write('%d\n' % self.pid)
1027 else:
1028 f.write('%s %d %7s ' %
1029 (job_id_str, self.pid, _JobStateStr(self.state)))
1030 f.write(self.thunk.UserString())
1031 f.write('\n')
1032
1033 def AddStateChange(self, s):
1034 # type: (ChildStateChange) -> None
1035 self.state_changes.append(s)
1036
1037 def AddPipeToClose(self, r, w):
1038 # type: (int, int) -> None
1039 self.close_r = r
1040 self.close_w = w
1041
1042 def MaybeClosePipe(self):
1043 # type: () -> None
1044 if self.close_r != -1:
1045 posix.close(self.close_r)
1046 posix.close(self.close_w)
1047
1048 def StartProcess(self, why):
1049 # type: (trace_t) -> int
1050 """Start this process with fork(), handling redirects."""
1051 pid = posix.fork()
1052 if pid < 0:
1053 # When does this happen?
1054 e_die('Fatal error in posix.fork()')
1055
1056 elif pid == 0: # child
1057 # Note: this happens in BOTH interactive and non-interactive shells.
1058 # We technically don't need to do most of it in non-interactive, since we
1059 # did not change state in InitInteractiveShell().
1060
1061 for st in self.state_changes:
1062 st.Apply()
1063
1064 # Python sets SIGPIPE handler to SIG_IGN by default. Child processes
1065 # shouldn't have this.
1066 # https://docs.python.org/2/library/signal.html
1067 # See Python/pythonrun.c.
1068 pyos.Sigaction(SIGPIPE, SIG_DFL)
1069
1070 # Respond to Ctrl-\ (core dump)
1071 pyos.Sigaction(SIGQUIT, SIG_DFL)
1072
1073 # Only standalone children should get Ctrl-Z. Pipelines remain in the
1074 # foreground because suspending them is difficult with our 'lastpipe'
1075 # semantics.
1076 pid = posix.getpid()
1077 if posix.getpgid(0) == pid and self.parent_pipeline is None:
1078 pyos.Sigaction(SIGTSTP, SIG_DFL)
1079
1080 # More signals from
1081 # https://www.gnu.org/software/libc/manual/html_node/Launching-Jobs.html
1082 # (but not SIGCHLD)
1083 pyos.Sigaction(SIGTTOU, SIG_DFL)
1084 pyos.Sigaction(SIGTTIN, SIG_DFL)
1085
1086 self.tracer.OnNewProcess(pid)
1087 # clear foreground pipeline for subshells
1088 self.thunk.Run()
1089 # Never returns
1090
1091 #log('STARTED process %s, pid = %d', self, pid)
1092 self.tracer.OnProcessStart(pid, why)
1093
1094 # Class invariant: after the process is started, it stores its PID.
1095 self.pid = pid
1096
1097 # SetPgid needs to be applied from the child and the parent to avoid
1098 # racing in calls to tcsetpgrp() in the parent. See APUE sec. 9.2.
1099 for st in self.state_changes:
1100 st.ApplyFromParent(self)
1101
1102 # Program invariant: We keep track of every child process!
1103 self.job_list.AddChildProcess(pid, self)
1104
1105 return pid
1106
1107 def Wait(self, waiter):
1108 # type: (Waiter) -> int
1109 """Wait for this process to finish."""
1110 while self.state == job_state_e.Running:
1111 # Only return if there's nothing to wait for. Keep waiting if we were
1112 # interrupted with a signal.
1113 if waiter.WaitForOne() == W1_ECHILD:
1114 break
1115
1116 assert self.status >= 0, self.status
1117 return self.status
1118
1119 def JobWait(self, waiter):
1120 # type: (Waiter) -> wait_status_t
1121 # wait builtin can be interrupted
1122 while self.state == job_state_e.Running:
1123 result = waiter.WaitForOne()
1124
1125 if result >= 0: # signal
1126 return wait_status.Cancelled(result)
1127
1128 if result == W1_ECHILD:
1129 break
1130
1131 return wait_status.Proc(self.status)
1132
1133 def WhenStopped(self, stop_sig):
1134 # type: (int) -> None
1135
1136 # 128 is a shell thing
1137 # https://www.gnu.org/software/bash/manual/html_node/Exit-Status.html
1138 self.status = 128 + stop_sig
1139 self.state = job_state_e.Stopped
1140
1141 if self.job_id == -1:
1142 # This process was started in the foreground
1143 self.job_list.AddJob(self)
1144
1145 if not self.in_background:
1146 self.job_control.MaybeTakeTerminal()
1147 self.SetBackground()
1148
1149 def WhenDone(self, pid, status):
1150 # type: (int, int) -> None
1151 """Called by the Waiter when this Process finishes."""
1152
1153 #log('Process WhenDone %d %d', pid, status)
1154 assert pid == self.pid, 'Expected %d, got %d' % (self.pid, pid)
1155 self.status = status
1156 self.state = job_state_e.Done
1157 if self.parent_pipeline:
1158 self.parent_pipeline.WhenDone(pid, status)
1159 else:
1160 if self.job_id != -1:
1161 # Job might have been brought to the foreground after being
1162 # assigned a job ID.
1163 if self.in_background:
1164 # TODO: bash only prints this interactively
1165 print_stderr('[%%%d] PID %d Done' %
1166 (self.job_id, self.pid))
1167
1168 self.job_list.RemoveJob(self.job_id)
1169
1170 self.job_list.RemoveChildProcess(self.pid)
1171
1172 if not self.in_background:
1173 self.job_control.MaybeTakeTerminal()
1174
1175 def RunProcess(self, waiter, why):
1176 # type: (Waiter, trace_t) -> int
1177 """Run this process synchronously."""
1178 self.StartProcess(why)
1179 # ShellExecutor might be calling this for the last part of a pipeline.
1180 if self.parent_pipeline is None:
1181 # QUESTION: Can the PGID of a single process just be the PID? i.e. avoid
1182 # calling getpgid()?
1183 self.job_control.MaybeGiveTerminal(posix.getpgid(self.pid))
1184 return self.Wait(waiter)
1185
1186
1187class ctx_Pipe(object):
1188
1189 def __init__(self, fd_state, fd, err_out):
1190 # type: (FdState, int, List[error.IOError_OSError]) -> None
1191 fd_state.PushStdinFromPipe(fd)
1192 self.fd_state = fd_state
1193 self.err_out = err_out
1194
1195 def __enter__(self):
1196 # type: () -> None
1197 pass
1198
1199 def __exit__(self, type, value, traceback):
1200 # type: (Any, Any, Any) -> None
1201 self.fd_state.Pop(self.err_out)
1202
1203
1204class Pipeline(Job):
1205 """A pipeline of processes to run.
1206
1207 Cases we handle:
1208
1209 foo | bar
1210 $(foo | bar)
1211 foo | bar | read v
1212 """
1213
1214 def __init__(self, sigpipe_status_ok, job_control, job_list, tracer):
1215 # type: (bool, JobControl, JobList, dev.Tracer) -> None
1216 Job.__init__(self)
1217 self.job_control = job_control
1218 self.job_list = job_list
1219 self.tracer = tracer
1220
1221 self.procs = [] # type: List[Process]
1222 self.pids = [] # type: List[int] # pids in order
1223 self.pipe_status = [] # type: List[int] # status in order
1224 self.status = -1 # for 'wait' jobs
1225
1226 self.pgid = INVALID_PGID
1227
1228 # Optional for foreground
1229 self.last_thunk = None # type: Tuple[CommandEvaluator, command_t]
1230 self.last_pipe = None # type: Tuple[int, int]
1231
1232 self.sigpipe_status_ok = sigpipe_status_ok
1233
1234 def ProcessGroupId(self):
1235 # type: () -> int
1236 """Returns the group ID of this pipeline."""
1237 return self.pgid
1238
1239 def DisplayJob(self, job_id, f, style):
1240 # type: (int, mylib.Writer, int) -> None
1241 if style == STYLE_PID_ONLY:
1242 f.write('%d\n' % self.procs[0].pid)
1243 else:
1244 # Note: this is STYLE_LONG.
1245 for i, proc in enumerate(self.procs):
1246 if i == 0: # show job ID for first element in pipeline
1247 job_id_str = '%%%d' % job_id
1248 else:
1249 job_id_str = ' ' # 2 spaces
1250
1251 f.write('%s %d %7s ' %
1252 (job_id_str, proc.pid, _JobStateStr(proc.state)))
1253 f.write(proc.thunk.UserString())
1254 f.write('\n')
1255
1256 def DebugPrint(self):
1257 # type: () -> None
1258 print('Pipeline in state %s' % _JobStateStr(self.state))
1259 if mylib.PYTHON: # %s for Process not allowed in C++
1260 for proc in self.procs:
1261 print(' proc %s' % proc)
1262 _, last_node = self.last_thunk
1263 print(' last %s' % last_node)
1264 print(' pipe_status %s' % self.pipe_status)
1265
1266 def Add(self, p):
1267 # type: (Process) -> None
1268 """Append a process to the pipeline."""
1269 if len(self.procs) == 0:
1270 self.procs.append(p)
1271 return
1272
1273 r, w = posix.pipe()
1274 #log('pipe for %s: %d %d', p, r, w)
1275 prev = self.procs[-1]
1276
1277 prev.AddStateChange(StdoutToPipe(r, w)) # applied on StartPipeline()
1278 p.AddStateChange(StdinFromPipe(r, w)) # applied on StartPipeline()
1279
1280 p.AddPipeToClose(r, w) # MaybeClosePipe() on StartPipeline()
1281
1282 self.procs.append(p)
1283
1284 def AddLast(self, thunk):
1285 # type: (Tuple[CommandEvaluator, command_t]) -> None
1286 """Append the last noden to the pipeline.
1287
1288 This is run in the CURRENT process. It is OPTIONAL, because
1289 pipelines in the background are run uniformly.
1290 """
1291 self.last_thunk = thunk
1292
1293 assert len(self.procs) != 0
1294
1295 r, w = posix.pipe()
1296 prev = self.procs[-1]
1297 prev.AddStateChange(StdoutToPipe(r, w))
1298
1299 self.last_pipe = (r, w) # So we can connect it to last_thunk
1300
1301 def StartPipeline(self, waiter):
1302 # type: (Waiter) -> None
1303
1304 # If we are creating a pipeline in a subshell or we aren't running with job
1305 # control, our children should remain in our inherited process group.
1306 # the pipelines's group ID.
1307 if self.job_control.Enabled():
1308 self.pgid = OWN_LEADER # first process in pipeline is the leader
1309
1310 for i, proc in enumerate(self.procs):
1311 if self.pgid != INVALID_PGID:
1312 proc.AddStateChange(SetPgid(self.pgid, self.tracer))
1313
1314 # Figure out the pid
1315 pid = proc.StartProcess(trace.PipelinePart)
1316 if i == 0 and self.pgid != INVALID_PGID:
1317 # Mimic bash and use the PID of the FIRST process as the group for the
1318 # whole pipeline.
1319 self.pgid = pid
1320
1321 self.pids.append(pid)
1322 self.pipe_status.append(-1) # uninitialized
1323
1324 # NOTE: This is done in the SHELL PROCESS after every fork() call.
1325 # It can't be done at the end; otherwise processes will have descriptors
1326 # from non-adjacent pipes.
1327 proc.MaybeClosePipe()
1328
1329 if self.last_thunk:
1330 self.pipe_status.append(-1) # for self.last_thunk
1331
1332 def LastPid(self):
1333 # type: () -> int
1334 """For the odd $! variable.
1335
1336 It would be better if job IDs or PGIDs were used consistently.
1337 """
1338 return self.pids[-1]
1339
1340 def Wait(self, waiter):
1341 # type: (Waiter) -> List[int]
1342 """Wait for this pipeline to finish."""
1343
1344 assert self.procs, "no procs for Wait()"
1345 # waitpid(-1) zero or more times
1346 while self.state == job_state_e.Running:
1347 # Keep waiting until there's nothing to wait for.
1348 if waiter.WaitForOne() == W1_ECHILD:
1349 break
1350
1351 return self.pipe_status
1352
1353 def JobWait(self, waiter):
1354 # type: (Waiter) -> wait_status_t
1355 """Called by 'wait' builtin, e.g. 'wait %1'."""
1356 # wait builtin can be interrupted
1357 assert self.procs, "no procs for Wait()"
1358 while self.state == job_state_e.Running:
1359 result = waiter.WaitForOne()
1360
1361 if result >= 0: # signal
1362 return wait_status.Cancelled(result)
1363
1364 if result == W1_ECHILD:
1365 break
1366
1367 return wait_status.Pipeline(self.pipe_status)
1368
1369 def RunLastPart(self, waiter, fd_state):
1370 # type: (Waiter, FdState) -> List[int]
1371 """Run this pipeline synchronously (foreground pipeline).
1372
1373 Returns:
1374 pipe_status (list of integers).
1375 """
1376 assert len(self.pids) == len(self.procs)
1377
1378 # TODO: break circular dep. Bit flags could go in ASDL or headers.
1379 from osh import cmd_eval
1380
1381 # This is tcsetpgrp()
1382 # TODO: fix race condition -- I believe the first process could have
1383 # stopped already, and thus getpgid() will fail
1384 self.job_control.MaybeGiveTerminal(self.pgid)
1385
1386 # Run the last part of the pipeline IN PARALLEL with other processes. It
1387 # may or may not fork:
1388 # echo foo | read line # no fork, the builtin runs in THIS shell process
1389 # ls | wc -l # fork for 'wc'
1390
1391 cmd_ev, last_node = self.last_thunk
1392
1393 assert self.last_pipe is not None
1394 r, w = self.last_pipe # set in AddLast()
1395 posix.close(w) # we will not write here
1396
1397 # Fix lastpipe / job control / DEBUG trap interaction
1398 cmd_flags = cmd_eval.NoDebugTrap if self.job_control.Enabled() else 0
1399
1400 # The ERR trap only runs for the WHOLE pipeline, not the COMPONENTS in
1401 # a pipeline.
1402 cmd_flags |= cmd_eval.NoErrTrap
1403
1404 io_errors = [] # type: List[error.IOError_OSError]
1405 with ctx_Pipe(fd_state, r, io_errors):
1406 cmd_ev.ExecuteAndCatch(last_node, cmd_flags)
1407
1408 if len(io_errors):
1409 e_die('Error setting up last part of pipeline: %s' %
1410 pyutil.strerror(io_errors[0]))
1411
1412 # We won't read anymore. If we don't do this, then 'cat' in 'cat
1413 # /dev/urandom | sleep 1' will never get SIGPIPE.
1414 posix.close(r)
1415
1416 self.pipe_status[-1] = cmd_ev.LastStatus()
1417 if self.AllDone():
1418 self.state = job_state_e.Done
1419
1420 #log('pipestatus before all have finished = %s', self.pipe_status)
1421 return self.Wait(waiter)
1422
1423 def AllDone(self):
1424 # type: () -> bool
1425
1426 # mycpp rewrite: all(status != -1 for status in self.pipe_status)
1427 for status in self.pipe_status:
1428 if status == -1:
1429 return False
1430 return True
1431
1432 def WhenDone(self, pid, status):
1433 # type: (int, int) -> None
1434 """Called by Process.WhenDone."""
1435 #log('Pipeline WhenDone %d %d', pid, status)
1436 i = self.pids.index(pid)
1437 assert i != -1, 'Unexpected PID %d' % pid
1438
1439 if status == 141 and self.sigpipe_status_ok:
1440 status = 0
1441
1442 self.job_list.RemoveChildProcess(pid)
1443 self.pipe_status[i] = status
1444 if self.AllDone():
1445 if self.job_id != -1:
1446 # Job might have been brought to the foreground after being
1447 # assigned a job ID.
1448 if self.in_background:
1449 print_stderr('[%%%d] PGID %d Done' %
1450 (self.job_id, self.pids[0]))
1451
1452 self.job_list.RemoveJob(self.job_id)
1453
1454 # status of pipeline is status of last process
1455 self.status = self.pipe_status[-1]
1456 self.state = job_state_e.Done
1457 if not self.in_background:
1458 self.job_control.MaybeTakeTerminal()
1459
1460
1461def _JobStateStr(i):
1462 # type: (job_state_t) -> str
1463 return job_state_str(i)[10:] # remove 'job_state.'
1464
1465
1466def _GetTtyFd():
1467 # type: () -> int
1468 """Returns -1 if stdio is not a TTY."""
1469 try:
1470 return posix.open("/dev/tty", O_NONBLOCK | O_NOCTTY | O_RDWR, 0o666)
1471 except (IOError, OSError) as e:
1472 return -1
1473
1474
1475class ctx_TerminalControl(object):
1476
1477 def __init__(self, job_control, errfmt):
1478 # type: (JobControl, ui.ErrorFormatter) -> None
1479 job_control.InitJobControl()
1480 self.job_control = job_control
1481 self.errfmt = errfmt
1482
1483 def __enter__(self):
1484 # type: () -> None
1485 pass
1486
1487 def __exit__(self, type, value, traceback):
1488 # type: (Any, Any, Any) -> None
1489
1490 # Return the TTY to the original owner before exiting.
1491 try:
1492 self.job_control.MaybeReturnTerminal()
1493 except error.FatalRuntime as e:
1494 # Don't abort the shell on error, just print a message.
1495 self.errfmt.PrettyPrintError(e)
1496
1497
1498class JobControl(object):
1499 """Interface to setpgid(), tcsetpgrp(), etc."""
1500
1501 def __init__(self):
1502 # type: () -> None
1503
1504 # The main shell's PID and group ID.
1505 self.shell_pid = -1
1506 self.shell_pgid = -1
1507
1508 # The fd of the controlling tty. Set to -1 when job control is disabled.
1509 self.shell_tty_fd = -1
1510
1511 # For giving the terminal back to our parent before exiting (if not a login
1512 # shell).
1513 self.original_tty_pgid = -1
1514
1515 def InitJobControl(self):
1516 # type: () -> None
1517 self.shell_pid = posix.getpid()
1518 orig_shell_pgid = posix.getpgid(0)
1519 self.shell_pgid = orig_shell_pgid
1520 self.shell_tty_fd = _GetTtyFd()
1521
1522 # If we aren't the leader of our process group, create a group and mark
1523 # ourselves as the leader.
1524 if self.shell_pgid != self.shell_pid:
1525 try:
1526 posix.setpgid(self.shell_pid, self.shell_pid)
1527 self.shell_pgid = self.shell_pid
1528 except (IOError, OSError) as e:
1529 self.shell_tty_fd = -1
1530
1531 if self.shell_tty_fd != -1:
1532 self.original_tty_pgid = posix.tcgetpgrp(self.shell_tty_fd)
1533
1534 # If stdio is a TTY, put the shell's process group in the foreground.
1535 try:
1536 posix.tcsetpgrp(self.shell_tty_fd, self.shell_pgid)
1537 except (IOError, OSError) as e:
1538 # We probably aren't in the session leader's process group. Disable job
1539 # control.
1540 self.shell_tty_fd = -1
1541 self.shell_pgid = orig_shell_pgid
1542 posix.setpgid(self.shell_pid, self.shell_pgid)
1543
1544 def Enabled(self):
1545 # type: () -> bool
1546 """
1547 Only the main shell process should bother with job control functions.
1548 """
1549 #log('ENABLED? %d', self.shell_tty_fd)
1550
1551 # TODO: get rid of getpid()? I think SubProgramThunk should set a
1552 # flag.
1553 return self.shell_tty_fd != -1 and posix.getpid() == self.shell_pid
1554
1555 # TODO: This isn't a PID. This is a process group ID?
1556 #
1557 # What should the table look like?
1558 #
1559 # Do we need the last PID? I don't know why bash prints that. Probably so
1560 # you can do wait $!
1561 # wait -n waits for any node to go from job_state_e.Running to job_state_e.Done?
1562 #
1563 # And it needs a flag for CURRENT, for the implicit arg to 'fg'.
1564 # job_id is just an integer. This is sort of lame.
1565 #
1566 # [job_id, flag, pgid, job_state, node]
1567
1568 def MaybeGiveTerminal(self, pgid):
1569 # type: (int) -> None
1570 """If stdio is a TTY, move the given process group to the
1571 foreground."""
1572 if not self.Enabled():
1573 # Only call tcsetpgrp when job control is enabled.
1574 return
1575
1576 try:
1577 posix.tcsetpgrp(self.shell_tty_fd, pgid)
1578 except (IOError, OSError) as e:
1579 e_die('osh: Failed to move process group %d to foreground: %s' %
1580 (pgid, pyutil.strerror(e)))
1581
1582 def MaybeTakeTerminal(self):
1583 # type: () -> None
1584 """If stdio is a TTY, return the main shell's process group to the
1585 foreground."""
1586 self.MaybeGiveTerminal(self.shell_pgid)
1587
1588 def MaybeReturnTerminal(self):
1589 # type: () -> None
1590 """Called before the shell exits."""
1591 self.MaybeGiveTerminal(self.original_tty_pgid)
1592
1593
1594class JobList(object):
1595 """Global list of jobs, used by a few builtins."""
1596
1597 def __init__(self):
1598 # type: () -> None
1599
1600 # job_id -> Job instance
1601 self.jobs = {} # type: Dict[int, Job]
1602
1603 # pid -> Process. This is for STOP notification.
1604 self.child_procs = {} # type: Dict[int, Process]
1605 self.debug_pipelines = [] # type: List[Pipeline]
1606
1607 # Counter used to assign IDs to jobs. It is incremented every time a job
1608 # is created. Once all active jobs are done it is reset to 1. I'm not
1609 # sure if this reset behavior is mandated by POSIX, but other shells do
1610 # it, so we mimic for the sake of compatibility.
1611 self.job_id = 1
1612
1613 def AddJob(self, job):
1614 # type: (Job) -> int
1615 """Add a background job to the list.
1616
1617 A job is either a Process or Pipeline. You can resume a job with 'fg',
1618 kill it with 'kill', etc.
1619
1620 Two cases:
1621
1622 1. async jobs: sleep 5 | sleep 4 &
1623 2. stopped jobs: sleep 5; then Ctrl-Z
1624 """
1625 job_id = self.job_id
1626 self.jobs[job_id] = job
1627 job.job_id = job_id
1628 self.job_id += 1
1629 return job_id
1630
1631 def RemoveJob(self, job_id):
1632 # type: (int) -> None
1633 """Process and Pipeline can call this."""
1634 mylib.dict_erase(self.jobs, job_id)
1635
1636 if len(self.jobs) == 0:
1637 self.job_id = 1
1638
1639 def AddChildProcess(self, pid, proc):
1640 # type: (int, Process) -> None
1641 """Every child process should be added here as soon as we know its PID.
1642
1643 When the Waiter gets an EXITED or STOPPED notification, we need
1644 to know about it so 'jobs' can work.
1645 """
1646 self.child_procs[pid] = proc
1647
1648 def RemoveChildProcess(self, pid):
1649 # type: (int) -> None
1650 """Remove the child process with the given PID."""
1651 mylib.dict_erase(self.child_procs, pid)
1652
1653 if mylib.PYTHON:
1654
1655 def AddPipeline(self, pi):
1656 # type: (Pipeline) -> None
1657 """For debugging only."""
1658 self.debug_pipelines.append(pi)
1659
1660 def ProcessFromPid(self, pid):
1661 # type: (int) -> Process
1662 """For wait $PID.
1663
1664 There's no way to wait for a pipeline with a PID. That uses job
1665 syntax, e.g. %1. Not a great interface.
1666 """
1667 return self.child_procs.get(pid)
1668
1669 def GetCurrentAndPreviousJobs(self):
1670 # type: () -> Tuple[Optional[Job], Optional[Job]]
1671 """Return the "current" and "previous" jobs (AKA `%+` and `%-`).
1672
1673 See the POSIX specification for the `jobs` builtin for details:
1674 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1675
1676 IMPORTANT NOTE: This method assumes that the jobs list will not change
1677 during its execution! This assumption holds for now because we only ever
1678 update the jobs list from the main loop after WaitPid() informs us of a
1679 change. If we implement `set -b` and install a signal handler for
1680 SIGCHLD we should be careful to synchronize it with this function. The
1681 unsafety of mutating GC data structures from a signal handler should
1682 make this a non-issue, but if bugs related to this appear this note may
1683 be helpful...
1684 """
1685 # Split all active jobs by state and sort each group by decreasing job
1686 # ID to approximate newness.
1687 stopped_jobs = [] # type: List[Job]
1688 running_jobs = [] # type: List[Job]
1689 for i in xrange(0, self.job_id):
1690 job = self.jobs.get(i, None)
1691 if not job:
1692 continue
1693
1694 if job.state == job_state_e.Stopped:
1695 stopped_jobs.append(job)
1696
1697 elif job.state == job_state_e.Running:
1698 running_jobs.append(job)
1699
1700 current = None # type: Optional[Job]
1701 previous = None # type: Optional[Job]
1702 # POSIX says: If there is any suspended job, then the current job shall
1703 # be a suspended job. If there are at least two suspended jobs, then the
1704 # previous job also shall be a suspended job.
1705 #
1706 # So, we will only return running jobs from here if there are no recent
1707 # stopped jobs.
1708 if len(stopped_jobs) > 0:
1709 current = stopped_jobs.pop()
1710
1711 if len(stopped_jobs) > 0:
1712 previous = stopped_jobs.pop()
1713
1714 if len(running_jobs) > 0 and not current:
1715 current = running_jobs.pop()
1716
1717 if len(running_jobs) > 0 and not previous:
1718 previous = running_jobs.pop()
1719
1720 if not previous:
1721 previous = current
1722
1723 return current, previous
1724
1725 def GetJobWithSpec(self, job_spec):
1726 # type: (str) -> Optional[Job]
1727 """Parse the given job spec and return the matching job. If there is no
1728 matching job, this function returns None.
1729
1730 See the POSIX spec for the `jobs` builtin for details about job specs:
1731 https://pubs.opengroup.org/onlinepubs/007904875/utilities/jobs.html
1732 """
1733 if job_spec in CURRENT_JOB_SPECS:
1734 current, _ = self.GetCurrentAndPreviousJobs()
1735 return current
1736
1737 if job_spec == '%-':
1738 _, previous = self.GetCurrentAndPreviousJobs()
1739 return previous
1740
1741 # TODO: Add support for job specs based on prefixes of process argv.
1742 m = util.RegexSearch(r'^%([0-9]+)$', job_spec)
1743 if m is not None:
1744 assert len(m) == 2
1745 job_id = int(m[1])
1746 if job_id in self.jobs:
1747 return self.jobs[job_id]
1748
1749 return None
1750
1751 def DisplayJobs(self, style):
1752 # type: (int) -> None
1753 """Used by the 'jobs' builtin.
1754
1755 https://pubs.opengroup.org/onlinepubs/9699919799/utilities/jobs.html
1756
1757 "By default, the jobs utility shall display the status of all stopped jobs,
1758 running background jobs and all jobs whose status has changed and have not
1759 been reported by the shell."
1760 """
1761 # NOTE: A job is a background process or pipeline.
1762 #
1763 # echo hi | wc -l -- this starts two processes. Wait for TWO
1764 # echo hi | wc -l & -- this starts a process which starts two processes
1765 # Wait for ONE.
1766 #
1767 # 'jobs -l' GROUPS the PIDs by job. It has the job number, + - indicators
1768 # for %% and %-, PID, status, and "command".
1769 #
1770 # Every component of a pipeline is on the same line with 'jobs', but
1771 # they're separated into different lines with 'jobs -l'.
1772 #
1773 # See demo/jobs-builtin.sh
1774
1775 # $ jobs -l
1776 # [1]+ 24414 Stopped sleep 5
1777 # 24415 | sleep 5
1778 # [2] 24502 Running sleep 6
1779 # 24503 | sleep 6
1780 # 24504 | sleep 5 &
1781 # [3]- 24508 Running sleep 6
1782 # 24509 | sleep 6
1783 # 24510 | sleep 5 &
1784
1785 f = mylib.Stdout()
1786 for job_id, job in iteritems(self.jobs):
1787 # Use the %1 syntax
1788 job.DisplayJob(job_id, f, style)
1789
1790 def DebugPrint(self):
1791 # type: () -> None
1792
1793 f = mylib.Stdout()
1794 f.write('\n')
1795 f.write('[process debug info]\n')
1796
1797 for pid, proc in iteritems(self.child_procs):
1798 proc.DisplayJob(-1, f, STYLE_DEFAULT)
1799 #p = ' |' if proc.parent_pipeline else ''
1800 #print('%d %7s %s%s' % (pid, _JobStateStr(proc.state), proc.thunk.UserString(), p))
1801
1802 if len(self.debug_pipelines):
1803 f.write('\n')
1804 f.write('[pipeline debug info]\n')
1805 for pi in self.debug_pipelines:
1806 pi.DebugPrint()
1807
1808 def ListRecent(self):
1809 # type: () -> None
1810 """For jobs -n, which I think is also used in the interactive
1811 prompt."""
1812 pass
1813
1814 def NumRunning(self):
1815 # type: () -> int
1816 """Return the number of running jobs.
1817
1818 Used by 'wait' and 'wait -n'.
1819 """
1820 count = 0
1821 for _, job in iteritems(self.jobs): # mycpp rewrite: from itervalues()
1822 if job.State() == job_state_e.Running:
1823 count += 1
1824 return count
1825
1826
1827# Some WaitForOne() return values
1828W1_OK = -2 # waitpid(-1) returned
1829W1_ECHILD = -3 # no processes to wait for
1830W1_AGAIN = -4 # WNOHANG was passed and there were no state changes
1831
1832
1833class Waiter(object):
1834 """A capability to wait for processes.
1835
1836 This must be a singleton (and is because CommandEvaluator is a singleton).
1837
1838 Invariants:
1839 - Every child process is registered once
1840 - Every child process is waited for
1841
1842 Canonical example of why we need a GLOBAL waiter:
1843
1844 { sleep 3; echo 'done 3'; } &
1845 { sleep 4; echo 'done 4'; } &
1846
1847 # ... do arbitrary stuff ...
1848
1849 { sleep 1; exit 1; } | { sleep 2; exit 2; }
1850
1851 Now when you do wait() after starting the pipeline, you might get a pipeline
1852 process OR a background process! So you have to distinguish between them.
1853 """
1854
1855 def __init__(self, job_list, exec_opts, signal_safe, tracer):
1856 # type: (JobList, optview.Exec, pyos.SignalSafe, dev.Tracer) -> None
1857 self.job_list = job_list
1858 self.exec_opts = exec_opts
1859 self.signal_safe = signal_safe
1860 self.tracer = tracer
1861 self.last_status = 127 # wait -n error code
1862
1863 def WaitForOne(self, waitpid_options=0):
1864 # type: (int) -> int
1865 """Wait until the next process returns (or maybe Ctrl-C).
1866
1867 Returns:
1868 One of these negative numbers:
1869 W1_ECHILD Nothing to wait for
1870 W1_OK Caller should keep waiting
1871 UNTRAPPED_SIGWINCH
1872 Or
1873 result > 0 Signal that waitpid() was interrupted with
1874
1875 In the interactive shell, we return 0 if we get a Ctrl-C, so the caller
1876 will try again.
1877
1878 Callers:
1879 wait -n -- loop until there is one fewer process (TODO)
1880 wait -- loop until there are no processes
1881 wait $! -- loop until job state is Done (process or pipeline)
1882 Process::Wait() -- loop until Process state is done
1883 Pipeline::Wait() -- loop until Pipeline state is done
1884
1885 Comparisons:
1886 bash: jobs.c waitchld() Has a special case macro(!) CHECK_WAIT_INTR for
1887 the wait builtin
1888
1889 dash: jobs.c waitproc() uses sigfillset(), sigprocmask(), etc. Runs in a
1890 loop while (gotsigchld), but that might be a hack for System V!
1891
1892 Should we have a cleaner API like named posix::wait_for_one() ?
1893
1894 wait_result =
1895 ECHILD -- nothing to wait for
1896 | Done(int pid, int status) -- process done
1897 | EINTR(bool sigint) -- may or may not retry
1898 """
1899 pid, status = pyos.WaitPid(waitpid_options)
1900 if pid == 0: # WNOHANG passed, and no state changes
1901 return W1_AGAIN
1902 elif pid < 0: # error case
1903 err_num = status
1904 #log('waitpid() error => %d %s', e.errno, pyutil.strerror(e))
1905 if err_num == ECHILD:
1906 return W1_ECHILD # nothing to wait for caller should stop
1907 elif err_num == EINTR: # Bug #858 fix
1908 #log('WaitForOne() => %d', self.trap_state.GetLastSignal())
1909 return self.signal_safe.LastSignal() # e.g. 1 for SIGHUP
1910 else:
1911 # The signature of waitpid() means this shouldn't happen
1912 raise AssertionError()
1913
1914 # All child processes are supposed to be in this dict. But this may
1915 # legitimately happen if a grandchild outlives the child (its parent).
1916 # Then it is reparented under this process, so we might receive
1917 # notification of its exit, even though we didn't start it. We can't have
1918 # any knowledge of such processes, so print a warning.
1919 if pid not in self.job_list.child_procs:
1920 print_stderr("oils: PID %d Stopped, but osh didn't start it" % pid)
1921 return W1_OK
1922
1923 proc = self.job_list.child_procs[pid]
1924 if 0:
1925 self.job_list.DebugPrint()
1926
1927 if WIFSIGNALED(status):
1928 term_sig = WTERMSIG(status)
1929 status = 128 + term_sig
1930
1931 # Print newline after Ctrl-C.
1932 if term_sig == SIGINT:
1933 print('')
1934
1935 proc.WhenDone(pid, status)
1936
1937 elif WIFEXITED(status):
1938 status = WEXITSTATUS(status)
1939 #log('exit status: %s', status)
1940 proc.WhenDone(pid, status)
1941
1942 elif WIFSTOPPED(status):
1943 #status = WEXITSTATUS(status)
1944 stop_sig = WSTOPSIG(status)
1945
1946 print_stderr('')
1947 print_stderr('oils: PID %d Stopped with signal %d' %
1948 (pid, stop_sig))
1949 proc.WhenStopped(stop_sig)
1950
1951 else:
1952 raise AssertionError(status)
1953
1954 self.last_status = status # for wait -n
1955 self.tracer.OnProcessEnd(pid, status)
1956 return W1_OK
1957
1958 def PollNotifications(self):
1959 # type: () -> None
1960 """
1961 Process all pending state changes.
1962 """
1963 while self.WaitForOne(waitpid_options=WNOHANG) == W1_OK:
1964 continue