OILS / frontend / args.py View on Github | oilshell.org

659 lines, 342 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, tagswitch, iteritems
62
63_ = log
64
65from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
90 self.shopt_changes = [
91 ] # type: List[OptChange] # -O nullglob +O nullglob
92 self.show_options = False # 'set -o' without an argument
93 self.actions = [] # type: List[str] # for compgen -A
94 self.saw_double_dash = False # for set --
95 for name, v in iteritems(defaults):
96 self.Set(name, v)
97
98 def SetTrue(self, name):
99 # type: (str) -> None
100 self.Set(name, value.Bool(True))
101
102 def Set(self, name, val):
103 # type: (str, value_t) -> None
104
105 # debug-completion -> debug_completion
106 name = name.replace('-', '_')
107 self.attrs[name] = val
108
109 if 0:
110 # Backward compatibility!
111 with tagswitch(val) as case:
112 if case(value_e.Undef):
113 py_val = None # type: Any
114 elif case(value_e.Bool):
115 py_val = cast(value.Bool, val).b
116 elif case(value_e.Int):
117 py_val = cast(value.Int, val).i
118 elif case(value_e.Float):
119 py_val = cast(value.Float, val).f
120 elif case(value_e.Str):
121 py_val = cast(value.Str, val).s
122 else:
123 raise AssertionError(val)
124
125 setattr(self, name, py_val)
126
127 def __repr__(self):
128 # type: () -> str
129 return '<_Attributes %s>' % self.__dict__
130
131
132class Reader(object):
133 """Wrapper for argv.
134
135 Modified by both the parsing loop and various actions.
136
137 The caller of the flags parser can continue to use it after flag parsing is
138 done to get args.
139 """
140
141 def __init__(self, argv, locs=None):
142 # type: (List[str], Optional[List[CompoundWord]]) -> None
143 self.argv = argv
144 self.locs = locs
145 self.n = len(argv)
146 self.i = 0
147
148 def __repr__(self):
149 # type: () -> str
150 return '<args.Reader %r %d>' % (self.argv, self.i)
151
152 def Next(self):
153 # type: () -> None
154 """Advance."""
155 self.i += 1
156
157 def Peek(self):
158 # type: () -> Optional[str]
159 """Return the next token, or None if there are no more.
160
161 None is your SENTINEL for parsing.
162 """
163 if self.i >= self.n:
164 return None
165 else:
166 return self.argv[self.i]
167
168 def Peek2(self):
169 # type: () -> Tuple[Optional[str], loc_t]
170 """Return the next token, or None if there are no more.
171
172 None is your SENTINEL for parsing.
173 """
174 if self.i >= self.n:
175 return None, loc.Missing
176 else:
177 return self.argv[self.i], self.locs[self.i]
178
179 def ReadRequired(self, error_msg):
180 # type: (str) -> str
181 arg = self.Peek()
182 if arg is None:
183 # point at argv[0]
184 e_usage(error_msg, self._FirstLocation())
185 self.Next()
186 return arg
187
188 def ReadRequired2(self, error_msg):
189 # type: (str) -> Tuple[str, loc_t]
190 arg = self.Peek()
191 if arg is None:
192 # point at argv[0]
193 e_usage(error_msg, self._FirstLocation())
194 location = self.locs[self.i]
195 self.Next()
196 return arg, location
197
198 def Rest(self):
199 # type: () -> List[str]
200 """Return the rest of the arguments."""
201 return self.argv[self.i:]
202
203 def Rest2(self):
204 # type: () -> Tuple[List[str], List[CompoundWord]]
205 """Return the rest of the arguments."""
206 return self.argv[self.i:], self.locs[self.i:]
207
208 def AtEnd(self):
209 # type: () -> bool
210 return self.i >= self.n # must be >= and not ==
211
212 def _FirstLocation(self):
213 # type: () -> loc_t
214 if self.locs is not None and self.locs[0] is not None:
215 return self.locs[0]
216 else:
217 return loc.Missing
218
219 def Location(self):
220 # type: () -> loc_t
221 if self.locs is not None:
222 if self.i == self.n:
223 i = self.n - 1 # if the last arg is missing, point at the one before
224 else:
225 i = self.i
226 if self.locs[i] is not None:
227 return self.locs[i]
228 else:
229 return loc.Missing
230 else:
231 return loc.Missing
232
233
234class _Action(object):
235 """What is done when a flag or option is detected."""
236
237 def __init__(self):
238 # type: () -> None
239 """Empty constructor for mycpp."""
240 pass
241
242 def OnMatch(self, attached_arg, arg_r, out):
243 # type: (Optional[str], Reader, _Attributes) -> bool
244 """Called when the flag matches.
245
246 Args:
247 prefix: '-' or '+'
248 suffix: ',' for -d,
249 arg_r: Reader() (rename to Input or InputReader?)
250 out: _Attributes() -- the thing we want to set
251
252 Returns:
253 True if flag parsing should be aborted.
254 """
255 raise NotImplementedError()
256
257
258class _ArgAction(_Action):
259
260 def __init__(self, name, quit_parsing_flags, valid=None):
261 # type: (str, bool, Optional[List[str]]) -> None
262 """
263 Args:
264 quit_parsing_flags: Stop parsing args after this one. for sh -c.
265 python -c behaves the same way.
266 """
267 self.name = name
268 self.quit_parsing_flags = quit_parsing_flags
269 self.valid = valid
270
271 def _Value(self, arg, location):
272 # type: (str, loc_t) -> value_t
273 raise NotImplementedError()
274
275 def OnMatch(self, attached_arg, arg_r, out):
276 # type: (Optional[str], Reader, _Attributes) -> bool
277 """Called when the flag matches."""
278 if attached_arg is not None: # for the ',' in -d,
279 arg = attached_arg
280 else:
281 arg_r.Next()
282 arg = arg_r.Peek()
283 if arg is None:
284 e_usage('expected argument to %r' % ('-' + self.name),
285 arg_r.Location())
286
287 val = self._Value(arg, arg_r.Location())
288 out.Set(self.name, val)
289 return self.quit_parsing_flags
290
291
292class SetToInt(_ArgAction):
293
294 def __init__(self, name):
295 # type: (str) -> None
296 # repeat defaults for C++ translation
297 _ArgAction.__init__(self, name, False, valid=None)
298
299 def _Value(self, arg, location):
300 # type: (str, loc_t) -> value_t
301 try:
302 i = mops.FromStr(arg)
303 except ValueError:
304 e_usage(
305 'expected integer after %s, got %r' % ('-' + self.name, arg),
306 location)
307
308 # So far all our int values are > 0, so use -1 as the 'unset' value
309 # corner case: this treats -0 as 0!
310 if mops.Greater(mops.BigInt(0), i):
311 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
312 location)
313 return value.Int(i)
314
315
316class SetToFloat(_ArgAction):
317
318 def __init__(self, name):
319 # type: (str) -> None
320 # repeat defaults for C++ translation
321 _ArgAction.__init__(self, name, False, valid=None)
322
323 def _Value(self, arg, location):
324 # type: (str, loc_t) -> value_t
325 try:
326 f = float(arg)
327 except ValueError:
328 e_usage(
329 'expected number after %r, got %r' % ('-' + self.name, arg),
330 location)
331 # So far all our float values are > 0, so use -1.0 as the 'unset' value
332 # corner case: this treats -0.0 as 0.0!
333 if f < 0:
334 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
335 location)
336 return value.Float(f)
337
338
339class SetToString(_ArgAction):
340
341 def __init__(self, name, quit_parsing_flags, valid=None):
342 # type: (str, bool, Optional[List[str]]) -> None
343 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
344
345 def _Value(self, arg, location):
346 # type: (str, loc_t) -> value_t
347 if self.valid is not None and arg not in self.valid:
348 e_usage(
349 'got invalid argument %r to %r, expected one of: %s' %
350 (arg, ('-' + self.name), '|'.join(self.valid)), location)
351 return value.Str(arg)
352
353
354class SetAttachedBool(_Action):
355 """This is the Go-like syntax of --verbose=1, --verbose, or --verbose=0."""
356
357 def __init__(self, name):
358 # type: (str) -> None
359 self.name = name
360
361 def OnMatch(self, attached_arg, arg_r, out):
362 # type: (Optional[str], Reader, _Attributes) -> bool
363 """Called when the flag matches."""
364
365 if attached_arg is not None: # '0' in --verbose=0
366 if attached_arg in ('0', 'F', 'false',
367 'False'): # TODO: incorrect translation
368 b = False
369 elif attached_arg in ('1', 'T', 'true', 'Talse'):
370 b = True
371 else:
372 e_usage(
373 'got invalid argument to boolean flag: %r' % attached_arg,
374 loc.Missing)
375 else:
376 b = True
377
378 out.Set(self.name, value.Bool(b))
379 return False
380
381
382class SetToTrue(_Action):
383
384 def __init__(self, name):
385 # type: (str) -> None
386 self.name = name
387
388 def OnMatch(self, attached_arg, arg_r, out):
389 # type: (Optional[str], Reader, _Attributes) -> bool
390 """Called when the flag matches."""
391 out.SetTrue(self.name)
392 return False
393
394
395class SetOption(_Action):
396 """Set an option to a boolean, for 'set +e'."""
397
398 def __init__(self, name):
399 # type: (str) -> None
400 self.name = name
401
402 def OnMatch(self, attached_arg, arg_r, out):
403 # type: (Optional[str], Reader, _Attributes) -> bool
404 """Called when the flag matches."""
405 b = (attached_arg == '-')
406 out.opt_changes.append((self.name, b))
407 return False
408
409
410class SetNamedOption(_Action):
411 """Set a named option to a boolean, for 'set +o errexit'."""
412
413 def __init__(self, shopt=False):
414 # type: (bool) -> None
415 self.names = [] # type: List[str]
416 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
417
418 def ArgName(self, name):
419 # type: (str) -> None
420 self.names.append(name)
421
422 def OnMatch(self, attached_arg, arg_r, out):
423 # type: (Optional[str], Reader, _Attributes) -> bool
424 """Called when the flag matches."""
425 b = (attached_arg == '-')
426 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
427 arg_r.Next() # always advance
428 arg = arg_r.Peek()
429 if arg is None:
430 # triggers on 'set -O' in addition to 'set -o' (meh OK)
431 out.show_options = True
432 return True # quit parsing
433
434 attr_name = arg # Note: validation is done elsewhere
435 if len(self.names) and attr_name not in self.names:
436 e_usage('Invalid option %r' % arg, loc.Missing)
437 changes = out.shopt_changes if self.shopt else out.opt_changes
438 changes.append((attr_name, b))
439 return False
440
441
442class SetAction(_Action):
443 """For compgen -f."""
444
445 def __init__(self, name):
446 # type: (str) -> None
447 self.name = name
448
449 def OnMatch(self, attached_arg, arg_r, out):
450 # type: (Optional[str], Reader, _Attributes) -> bool
451 out.actions.append(self.name)
452 return False
453
454
455class SetNamedAction(_Action):
456 """For compgen -A file."""
457
458 def __init__(self):
459 # type: () -> None
460 self.names = [] # type: List[str]
461
462 def ArgName(self, name):
463 # type: (str) -> None
464 self.names.append(name)
465
466 def OnMatch(self, attached_arg, arg_r, out):
467 # type: (Optional[str], Reader, _Attributes) -> bool
468 """Called when the flag matches."""
469 arg_r.Next() # always advance
470 arg = arg_r.Peek()
471 if arg is None:
472 e_usage('Expected argument for action', loc.Missing)
473
474 attr_name = arg
475 # Validate the option name against a list of valid names.
476 if len(self.names) and attr_name not in self.names:
477 e_usage('Invalid action name %r' % arg, loc.Missing)
478 out.actions.append(attr_name)
479 return False
480
481
482def Parse(spec, arg_r):
483 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
484
485 # NOTE about -:
486 # 'set -' ignores it, vs set
487 # 'unset -' or 'export -' seems to treat it as a variable name
488 out = _Attributes(spec.defaults)
489
490 while not arg_r.AtEnd():
491 arg = arg_r.Peek()
492 if arg == '--':
493 out.saw_double_dash = True
494 arg_r.Next()
495 break
496
497 # Only accept -- if there are any long flags defined
498 if len(spec.actions_long) and arg.startswith('--'):
499 pos = arg.find('=', 2)
500 if pos == -1:
501 suffix = None # type: Optional[str]
502 flag_name = arg[2:] # strip off --
503 else:
504 suffix = arg[pos + 1:]
505 flag_name = arg[2:pos]
506
507 action = spec.actions_long.get(flag_name)
508 if action is None:
509 e_usage('got invalid flag %r' % arg, arg_r.Location())
510
511 action.OnMatch(suffix, arg_r, out)
512 arg_r.Next()
513 continue
514
515 elif arg.startswith('-') and len(arg) > 1:
516 n = len(arg)
517 for i in xrange(1, n): # parse flag combos like -rx
518 ch = arg[i]
519
520 if ch == '0':
521 ch = 'Z' # hack for read -0
522
523 if ch in spec.plus_flags:
524 out.Set(ch, value.Str('-'))
525 continue
526
527 if ch in spec.arity0: # e.g. read -r
528 out.SetTrue(ch)
529 continue
530
531 if ch in spec.arity1: # e.g. read -t1.0
532 action = spec.arity1[ch]
533 # make sure we don't pass empty string for read -t
534 attached_arg = arg[i + 1:] if i < n - 1 else None
535 action.OnMatch(attached_arg, arg_r, out)
536 break
537
538 e_usage("doesn't accept flag %s" % ('-' + ch),
539 arg_r.Location())
540
541 arg_r.Next() # next arg
542
543 # Only accept + if there are ANY options defined, e.g. for declare +rx.
544 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
545 n = len(arg)
546 for i in xrange(1, n): # parse flag combos like -rx
547 ch = arg[i]
548 if ch in spec.plus_flags:
549 out.Set(ch, value.Str('+'))
550 continue
551
552 e_usage("doesn't accept option %s" % ('+' + ch),
553 arg_r.Location())
554
555 arg_r.Next() # next arg
556
557 else: # a regular arg
558 break
559
560 return out
561
562
563def ParseLikeEcho(spec, arg_r):
564 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
565 """Echo is a special case. These work: echo -n echo -en.
566
567 - But don't respect --
568 - doesn't fail when an invalid flag is passed
569 """
570 out = _Attributes(spec.defaults)
571
572 while not arg_r.AtEnd():
573 arg = arg_r.Peek()
574 chars = arg[1:]
575 if arg.startswith('-') and len(chars):
576 # Check if it looks like -en or not. TODO: could optimize this.
577 done = False
578 for c in chars:
579 if c not in spec.arity0:
580 done = True
581 break
582 if done:
583 break
584
585 for ch in chars:
586 out.SetTrue(ch)
587
588 else:
589 break # Looks like an arg
590
591 arg_r.Next() # next arg
592
593 return out
594
595
596def ParseMore(spec, arg_r):
597 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
598 """Return attributes and an index.
599
600 Respects +, like set +eu
601
602 We do NOT respect:
603
604 WRONG: sh -cecho OK: sh -c echo
605 WRONG: set -opipefail OK: set -o pipefail
606
607 But we do accept these
608
609 set -euo pipefail
610 set -oeu pipefail
611 set -oo pipefail errexit
612 """
613 out = _Attributes(spec.defaults)
614
615 quit = False
616 while not arg_r.AtEnd():
617 arg = arg_r.Peek()
618 if arg == '--':
619 out.saw_double_dash = True
620 arg_r.Next()
621 break
622
623 if arg.startswith('--'):
624 action = spec.actions_long.get(arg[2:])
625 if action is None:
626 e_usage('got invalid flag %r' % arg, arg_r.Location())
627
628 # TODO: attached_arg could be 'bar' for --foo=bar
629 action.OnMatch(None, arg_r, out)
630 arg_r.Next()
631 continue
632
633 # corner case: sh +c is also accepted!
634 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
635 # note: we're not handling sh -cecho (no space) as an argument
636 # It complains about a missing argument
637
638 char0 = arg[0]
639
640 # TODO: set - - empty
641 for ch in arg[1:]:
642 #log('ch %r arg_r %s', ch, arg_r)
643 action = spec.actions_short.get(ch)
644 if action is None:
645 e_usage('got invalid flag %r' % ('-' + ch),
646 arg_r.Location())
647
648 attached_arg = char0 if ch in spec.plus_flags else None
649 quit = action.OnMatch(attached_arg, arg_r, out)
650 arg_r.Next() # process the next flag
651
652 if quit:
653 break
654 else:
655 continue
656
657 break # it's a regular arg
658
659 return out