OILS / frontend / args.py View on Github | oils.pub

698 lines, 349 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, iteritems
62
63_ = log
64
65from typing import (Tuple, Optional, Dict, List, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 # -o errexit +o nounset
90 self.opt_changes = [] # type: List[OptChange]
91
92 # -O nullglob +O nullglob
93 self.shopt_changes = [] # type: List[OptChange]
94
95 # Special case for --eval and --eval-pure? For bin/osh. It seems OK
96 # to have one now. The pool tells us if it was pure or not.
97 # Note: MAIN_SPEC is different than SET_SPEC, so set --eval does
98 # nothing.
99 self.eval_flags = [] # type: List[Tuple[str, bool]]
100
101 self.show_options = False # 'set -o' without an argument
102 self.actions = [] # type: List[str] # for compgen -A
103 self.saw_double_dash = False # for set --
104 for name, v in iteritems(defaults):
105 self.Set(name, v)
106
107 def SetTrue(self, name):
108 # type: (str) -> None
109 self.Set(name, value.Bool(True))
110
111 def Set(self, name, val):
112 # type: (str, value_t) -> None
113
114 # debug-completion -> debug_completion
115 name = name.replace('-', '_')
116
117 # similar hack to avoid C++ keyword in frontend/flag_gen.py
118 if name == 'extern':
119 name = 'extern_'
120 elif name == 'private':
121 name = 'private_'
122
123 self.attrs[name] = val
124
125 def __repr__(self):
126 # type: () -> str
127 return '<_Attributes %s>' % self.__dict__
128
129
130class Reader(object):
131 """Wrapper for argv.
132
133 Modified by both the parsing loop and various actions.
134
135 The caller of the flags parser can continue to use it after flag parsing is
136 done to get args.
137 """
138
139 def __init__(self, argv, locs=None):
140 # type: (List[str], Optional[List[CompoundWord]]) -> None
141 self.argv = argv
142 self.locs = locs
143 self.n = len(argv)
144 self.i = 0
145
146 def __repr__(self):
147 # type: () -> str
148 return '<args.Reader %r %d>' % (self.argv, self.i)
149
150 def Next(self):
151 # type: () -> None
152 """Advance."""
153 self.i += 1
154
155 def Peek(self):
156 # type: () -> Optional[str]
157 """Return the next token, or None if there are no more.
158
159 None is your SENTINEL for parsing.
160 """
161 if self.i >= self.n:
162 return None
163 else:
164 return self.argv[self.i]
165
166 def Peek2(self):
167 # type: () -> Tuple[Optional[str], loc_t]
168 """Return the next token, or None if there are no more.
169
170 None is your SENTINEL for parsing.
171 """
172 if self.i >= self.n:
173 return None, loc.Missing
174 else:
175 return self.argv[self.i], self.locs[self.i]
176
177 def ReadRequired(self, error_msg):
178 # type: (str) -> str
179 arg = self.Peek()
180 if arg is None:
181 # point at argv[0]
182 e_usage(error_msg, self._FirstLocation())
183 self.Next()
184 return arg
185
186 def ReadRequired2(self, error_msg):
187 # type: (str) -> Tuple[str, CompoundWord]
188 arg = self.Peek()
189 if arg is None:
190 # point at argv[0]
191 e_usage(error_msg, self._FirstLocation())
192 location = self.locs[self.i]
193 self.Next()
194 return arg, location
195
196 def Rest(self):
197 # type: () -> List[str]
198 """Return the rest of the arguments."""
199 return self.argv[self.i:]
200
201 def Rest2(self):
202 # type: () -> Tuple[List[str], List[CompoundWord]]
203 """Return the rest of the arguments."""
204 return self.argv[self.i:], self.locs[self.i:]
205
206 def AtEnd(self):
207 # type: () -> bool
208 return self.i >= self.n # must be >= and not ==
209
210 def Done(self):
211 # type: () -> None
212 if not self.AtEnd():
213 e_usage('got too many arguments', self.Location())
214
215 def _FirstLocation(self):
216 # type: () -> loc_t
217 if self.locs is not None and self.locs[0] is not None:
218 return self.locs[0]
219 else:
220 return loc.Missing
221
222 def Location(self):
223 # type: () -> loc_t
224 if self.locs is not None:
225 if self.i == self.n:
226 i = self.n - 1 # if the last arg is missing, point at the one before
227 else:
228 i = self.i
229 if self.locs[i] is not None:
230 return self.locs[i]
231 else:
232 return loc.Missing
233 else:
234 return loc.Missing
235
236
237class _Action(object):
238 """What is done when a flag or option is detected."""
239
240 def __init__(self):
241 # type: () -> None
242 """Empty constructor for mycpp."""
243 pass
244
245 def OnMatch(self, attached_arg, arg_r, out):
246 # type: (Optional[str], Reader, _Attributes) -> bool
247 """Called when the flag matches.
248
249 Args:
250 prefix: '-' or '+'
251 suffix: ',' for -d,
252 arg_r: Reader() (rename to Input or InputReader?)
253 out: _Attributes() -- the thing we want to set
254
255 Returns:
256 True if flag parsing should be aborted.
257 """
258 raise NotImplementedError()
259
260
261class AppendEvalFlag(_Action):
262
263 def __init__(self, name):
264 # type: (str) -> None
265 _Action.__init__(self)
266 self.name = name
267 self.is_pure = (name == 'eval-pure')
268
269 def OnMatch(self, attached_arg, arg_r, out):
270 # type: (Optional[str], Reader, _Attributes) -> bool
271 """Called when the flag matches."""
272
273 assert attached_arg is None, attached_arg
274
275 arg_r.Next()
276 arg = arg_r.Peek()
277 if arg is None:
278 e_usage('expected argument to %r' % ('--' + self.name),
279 arg_r.Location())
280
281 # is_pure is True for --eval-pure
282 out.eval_flags.append((arg, self.is_pure))
283 return False
284
285
286class _ArgAction(_Action):
287
288 def __init__(self, name, quit_parsing_flags, valid=None):
289 # type: (str, bool, Optional[List[str]]) -> None
290 """
291 Args:
292 quit_parsing_flags: Stop parsing args after this one. for sh -c.
293 python -c behaves the same way.
294 """
295 self.name = name
296 self.quit_parsing_flags = quit_parsing_flags
297 self.valid = valid
298
299 def _Value(self, arg, location):
300 # type: (str, loc_t) -> value_t
301 raise NotImplementedError()
302
303 def OnMatch(self, attached_arg, arg_r, out):
304 # type: (Optional[str], Reader, _Attributes) -> bool
305 """Called when the flag matches."""
306 if attached_arg is not None: # for the ',' in -d,
307 arg = attached_arg
308 else:
309 arg_r.Next()
310 arg = arg_r.Peek()
311 if arg is None:
312 e_usage('expected argument to %r' % ('-' + self.name),
313 arg_r.Location())
314
315 val = self._Value(arg, arg_r.Location())
316 out.Set(self.name, val)
317 return self.quit_parsing_flags
318
319
320class SetToInt(_ArgAction):
321
322 def __init__(self, name):
323 # type: (str) -> None
324 # repeat defaults for C++ translation
325 _ArgAction.__init__(self, name, False, valid=None)
326
327 def _Value(self, arg, location):
328 # type: (str, loc_t) -> value_t
329 #if match.LooksLikeInteger(arg):
330 if True: # break dependency for prebuilt/
331 ok, i = mops.FromStr2(arg)
332 if not ok:
333 #e_usage('Integer too big: %s' % arg, location)
334 e_usage(
335 'expected integer after %s, got %r' %
336 ('-' + self.name, arg), location)
337 else:
338 pass
339 #e_usage(
340 #'expected integer after %s, got %r' % ('-' + self.name, arg),
341 #location)
342
343 # So far all our int values are > 0, so use -1 as the 'unset' value
344 # corner case: this treats -0 as 0!
345 if mops.Greater(mops.BigInt(0), i):
346 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
347 location)
348 return value.Int(i)
349
350
351class SetToFloat(_ArgAction):
352
353 def __init__(self, name):
354 # type: (str) -> None
355 # repeat defaults for C++ translation
356 _ArgAction.__init__(self, name, False, valid=None)
357
358 def _Value(self, arg, location):
359 # type: (str, loc_t) -> value_t
360 try:
361 f = float(arg)
362 except ValueError:
363 e_usage(
364 'expected number after %r, got %r' % ('-' + self.name, arg),
365 location)
366 # So far all our float values are > 0, so use -1.0 as the 'unset' value
367 # corner case: this treats -0.0 as 0.0!
368 if f < 0:
369 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
370 location)
371 return value.Float(f)
372
373
374class SetToString(_ArgAction):
375
376 def __init__(self, name, quit_parsing_flags, valid=None):
377 # type: (str, bool, Optional[List[str]]) -> None
378 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
379
380 def _Value(self, arg, location):
381 # type: (str, loc_t) -> value_t
382 if self.valid is not None and arg not in self.valid:
383 e_usage(
384 'got invalid argument %r to %r, expected one of: %s' %
385 (arg, ('-' + self.name), '|'.join(self.valid)), location)
386 return value.Str(arg)
387
388
389class SetAttachedBool(_Action):
390
391 def __init__(self, name):
392 # type: (str) -> None
393 self.name = name
394
395 def OnMatch(self, attached_arg, arg_r, out):
396 # type: (Optional[str], Reader, _Attributes) -> bool
397 """Called when the flag matches."""
398
399 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
400 #
401 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
402 #
403 # 'attached_arg' is also used for -t0 though, which is weird
404
405 if attached_arg is not None: # '0' in --verbose=0
406 if attached_arg in ('0', 'F', 'false', 'False'):
407 b = False
408 elif attached_arg in ('1', 'T', 'true', 'True'):
409 b = True
410 else:
411 e_usage(
412 'got invalid argument to boolean flag: %r' % attached_arg,
413 loc.Missing)
414 else:
415 b = True
416
417 out.Set(self.name, value.Bool(b))
418 return False
419
420
421class SetToTrue(_Action):
422
423 def __init__(self, name):
424 # type: (str) -> None
425 self.name = name
426
427 def OnMatch(self, attached_arg, arg_r, out):
428 # type: (Optional[str], Reader, _Attributes) -> bool
429 """Called when the flag matches."""
430 out.SetTrue(self.name)
431 return False
432
433
434class SetOption(_Action):
435 """Set an option to a boolean, for 'set +e'."""
436
437 def __init__(self, name):
438 # type: (str) -> None
439 self.name = name
440
441 def OnMatch(self, attached_arg, arg_r, out):
442 # type: (Optional[str], Reader, _Attributes) -> bool
443 """Called when the flag matches."""
444 b = (attached_arg == '-')
445 out.opt_changes.append((self.name, b))
446 return False
447
448
449class SetNamedOption(_Action):
450 """Set a named option to a boolean, for 'set +o errexit'."""
451
452 def __init__(self, shopt=False):
453 # type: (bool) -> None
454 self.names = [] # type: List[str]
455 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
456
457 def ArgName(self, name):
458 # type: (str) -> None
459 self.names.append(name)
460
461 def OnMatch(self, attached_arg, arg_r, out):
462 # type: (Optional[str], Reader, _Attributes) -> bool
463 """Called when the flag matches."""
464 b = (attached_arg == '-')
465 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
466 arg_r.Next() # always advance
467 arg = arg_r.Peek()
468 if arg is None:
469 # triggers on 'set -O' in addition to 'set -o' (meh OK)
470 out.show_options = True
471 return True # quit parsing
472
473 attr_name = arg # Note: validation is done elsewhere
474 if len(self.names) and attr_name not in self.names:
475 e_usage('Invalid option %r' % arg, loc.Missing)
476 changes = out.shopt_changes if self.shopt else out.opt_changes
477 changes.append((attr_name, b))
478 return False
479
480
481class SetAction(_Action):
482 """For compgen -f."""
483
484 def __init__(self, name):
485 # type: (str) -> None
486 self.name = name
487
488 def OnMatch(self, attached_arg, arg_r, out):
489 # type: (Optional[str], Reader, _Attributes) -> bool
490 out.actions.append(self.name)
491 return False
492
493
494class SetNamedAction(_Action):
495 """For compgen -A file."""
496
497 def __init__(self):
498 # type: () -> None
499 self.names = [] # type: List[str]
500
501 def ArgName(self, name):
502 # type: (str) -> None
503 self.names.append(name)
504
505 def OnMatch(self, attached_arg, arg_r, out):
506 # type: (Optional[str], Reader, _Attributes) -> bool
507 """Called when the flag matches."""
508 arg_r.Next() # always advance
509 arg = arg_r.Peek()
510 if arg is None:
511 e_usage('Expected argument for action', loc.Missing)
512
513 attr_name = arg
514 # Validate the option name against a list of valid names.
515 if len(self.names) and attr_name not in self.names:
516 e_usage('Invalid action name %r' % arg, loc.Missing)
517 out.actions.append(attr_name)
518 return False
519
520
521def Parse(spec, arg_r):
522 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
523
524 # NOTE about -:
525 # 'set -' ignores it, vs set
526 # 'unset -' or 'export -' seems to treat it as a variable name
527 out = _Attributes(spec.defaults)
528
529 while not arg_r.AtEnd():
530 arg = arg_r.Peek()
531 if arg == '--':
532 out.saw_double_dash = True
533 arg_r.Next()
534 break
535
536 # Only accept -- if there are any long flags defined
537 if len(spec.actions_long) and arg.startswith('--'):
538 pos = arg.find('=', 2)
539 if pos == -1:
540 suffix = None # type: Optional[str]
541 flag_name = arg[2:] # strip off --
542 else:
543 suffix = arg[pos + 1:]
544 flag_name = arg[2:pos]
545
546 action = spec.actions_long.get(flag_name)
547 if action is None:
548 e_usage('got invalid flag %r' % arg, arg_r.Location())
549
550 action.OnMatch(suffix, arg_r, out)
551 arg_r.Next()
552 continue
553
554 elif arg.startswith('-') and len(arg) > 1:
555 n = len(arg)
556 for i in xrange(1, n): # parse flag combos like -rx
557 ch = arg[i]
558
559 if ch == '0':
560 ch = 'Z' # hack for read -0
561
562 if ch in spec.plus_flags:
563 out.Set(ch, value.Str('-'))
564 continue
565
566 if ch in spec.arity0: # e.g. read -r
567 out.SetTrue(ch)
568 continue
569
570 if ch in spec.arity1: # e.g. read -t1.0
571 action = spec.arity1[ch]
572 # make sure we don't pass empty string for read -t
573 attached_arg = arg[i + 1:] if i < n - 1 else None
574 action.OnMatch(attached_arg, arg_r, out)
575 break
576
577 e_usage("doesn't accept flag %s" % ('-' + ch),
578 arg_r.Location())
579
580 arg_r.Next() # next arg
581
582 # Only accept + if there are ANY options defined, e.g. for declare +rx.
583 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
584 n = len(arg)
585 for i in xrange(1, n): # parse flag combos like -rx
586 ch = arg[i]
587 if ch in spec.plus_flags:
588 out.Set(ch, value.Str('+'))
589 continue
590
591 e_usage("doesn't accept option %s" % ('+' + ch),
592 arg_r.Location())
593
594 arg_r.Next() # next arg
595
596 else: # a regular arg
597 break
598
599 return out
600
601
602def ParseLikeEcho(spec, arg_r):
603 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
604 """Echo is a special case. These work: echo -n echo -en.
605
606 - But don't respect --
607 - doesn't fail when an invalid flag is passed
608 """
609 out = _Attributes(spec.defaults)
610
611 while not arg_r.AtEnd():
612 arg = arg_r.Peek()
613 chars = arg[1:]
614 if arg.startswith('-') and len(chars):
615 # Check if it looks like -en or not. TODO: could optimize this.
616 done = False
617 for c in chars:
618 if c not in spec.arity0:
619 done = True
620 break
621 if done:
622 break
623
624 for ch in chars:
625 out.SetTrue(ch)
626
627 else:
628 break # Looks like an arg
629
630 arg_r.Next() # next arg
631
632 return out
633
634
635def ParseMore(spec, arg_r):
636 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
637 """Return attributes and an index.
638
639 Respects +, like set +eu
640
641 We do NOT respect:
642
643 WRONG: sh -cecho OK: sh -c echo
644 WRONG: set -opipefail OK: set -o pipefail
645
646 But we do accept these
647
648 set -euo pipefail
649 set -oeu pipefail
650 set -oo pipefail errexit
651 """
652 out = _Attributes(spec.defaults)
653
654 quit = False
655 while not arg_r.AtEnd():
656 arg = arg_r.Peek()
657 if arg == '--':
658 out.saw_double_dash = True
659 arg_r.Next()
660 break
661
662 if arg.startswith('--'):
663 action = spec.actions_long.get(arg[2:])
664 if action is None:
665 e_usage('got invalid flag %r' % arg, arg_r.Location())
666
667 # Note: not parsing --foo=bar as attached_arg, as above
668 action.OnMatch(None, arg_r, out)
669 arg_r.Next()
670 continue
671
672 # corner case: sh +c is also accepted!
673 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
674 # note: we're not handling sh -cecho (no space) as an argument
675 # It complains about a missing argument
676
677 char0 = arg[0]
678
679 # TODO: set - - empty
680 for ch in arg[1:]:
681 #log('ch %r arg_r %s', ch, arg_r)
682 action = spec.actions_short.get(ch)
683 if action is None:
684 e_usage('got invalid flag %r' % ('-' + ch),
685 arg_r.Location())
686
687 attached_arg = char0 if ch in spec.plus_flags else None
688 quit = action.OnMatch(attached_arg, arg_r, out)
689 arg_r.Next() # process the next flag
690
691 if quit:
692 break
693 else:
694 continue
695
696 break # it's a regular arg
697
698 return out