osh/word_eval.py

OILS / osh / word_eval.py View on Github | oilshell.org

2348 lines, 1420 significant

1	"""
2	word_eval.py - Evaluator for the word language.
3	"""
4
5	from _devbuild.gen.id_kind_asdl import Id, Kind, Kind_str
6	from _devbuild.gen.syntax_asdl import (
7	Token,
8	NameTok,
9	loc,
10	loc_t,
11	BracedVarSub,
12	CommandSub,
13	bracket_op,
14	bracket_op_e,
15	suffix_op,
16	suffix_op_e,
17	ShArrayLiteral,
18	SingleQuoted,
19	DoubleQuoted,
20	word_e,
21	word_t,
22	CompoundWord,
23	rhs_word,
24	rhs_word_e,
25	rhs_word_t,
26	word_part,
27	word_part_e,
28	)
29	from _devbuild.gen.runtime_asdl import (
30	part_value,
31	part_value_e,
32	part_value_t,
33	cmd_value,
34	cmd_value_e,
35	cmd_value_t,
36	AssignArg,
37	a_index,
38	a_index_e,
39	VTestPlace,
40	VarSubState,
41	)
42	from _devbuild.gen.option_asdl import option_i
43	from _devbuild.gen.value_asdl import (
44	value,
45	value_e,
46	value_t,
47	sh_lvalue,
48	sh_lvalue_t,
49	)
50	from core import error
51	from core import pyos
52	from core import pyutil
53	from core import state
54	from core import ui
55	from core import util
56	from data_lang import j8_lite
57	from core.error import e_die
58	from frontend import consts
59	from frontend import lexer
60	from frontend import location
61	from mycpp import mops
62	from mycpp.mylib import log, tagswitch, NewDict
63	from osh import braces
64	from osh import glob_
65	from osh import string_ops
66	from osh import word_
67	from osh import word_compile
68	from ysh import expr_eval
69	from ysh import val_ops
70
71	from typing import Optional, Tuple, List, Dict, cast, TYPE_CHECKING
72
73	if TYPE_CHECKING:
74	from _devbuild.gen.syntax_asdl import word_part_t
75	from _devbuild.gen.option_asdl import builtin_t
76	from core import optview
77	from core.state import Mem
78	from core.ui import ErrorFormatter
79	from core.vm import _Executor
80	from osh.split import SplitContext
81	from osh import prompt
82	from osh import sh_expr_eval
83
84	# Flags for _EvalWordToParts and _EvalWordPart (not all are used for both)
85	QUOTED = 1 << 0
86	IS_SUBST = 1 << 1
87
88	EXTGLOB_FILES = 1 << 2 # allow @(cc) from file system?
89	EXTGLOB_MATCH = 1 << 3 # allow @(cc) in pattern matching?
90	EXTGLOB_NESTED = 1 << 4 # for @(one\|!(two\|three))
91
92	# For EvalWordToString
93	QUOTE_FNMATCH = 1 << 5
94	QUOTE_ERE = 1 << 6
95
96	# For compatibility, ${BASH_SOURCE} and ${BASH_SOURCE[@]} are both valid.
97	# Ditto for ${FUNCNAME} and ${BASH_LINENO}.
98	_STRING_AND_ARRAY = ['BASH_SOURCE', 'FUNCNAME', 'BASH_LINENO']
99
100
101	def ShouldArrayDecay(var_name, exec_opts, is_plain_var_sub=True):
102	# type: (str, optview.Exec, bool) -> bool
103	"""Return whether we should allow ${a} to mean ${a[0]}."""
104	return (not exec_opts.strict_array() or
105	is_plain_var_sub and var_name in _STRING_AND_ARRAY)
106
107
108	def DecayArray(val):
109	# type: (value_t) -> value_t
110	"""Resolve ${array} to ${array[0]}."""
111	if val.tag() == value_e.BashArray:
112	array_val = cast(value.BashArray, val)
113	s = array_val.strs[0] if len(array_val.strs) else None
114	elif val.tag() == value_e.BashAssoc:
115	assoc_val = cast(value.BashAssoc, val)
116	s = assoc_val.d['0'] if '0' in assoc_val.d else None
117	else:
118	raise AssertionError(val.tag())
119
120	if s is None:
121	return value.Undef
122	else:
123	return value.Str(s)
124
125
126	def GetArrayItem(strs, index):
127	# type: (List[str], int) -> Optional[str]
128
129	n = len(strs)
130	if index < 0:
131	index += n
132
133	if 0 <= index and index < n:
134	# TODO: strs->index() has a redundant check for (i < 0)
135	s = strs[index]
136	# note: s could be None because representation is sparse
137	else:
138	s = None
139	return s
140
141
142	# Use libc to parse NAME, NAME=value, and NAME+=value. We want submatch
143	# extraction, but I haven't used that in re2c, and we would need a new kind of
144	# binding.
145	#
146	ASSIGN_ARG_RE = '^([a-zA-Z_][a-zA-Z0-9_])((=\|\+=)(.))?$'
147
148	# Eggex equivalent:
149	#
150	# VarName = /
151	# [a-z A-Z _ ]
152	# [a-z A-Z 0-9 _ ]*
153	# /
154	#
155	# SplitArg = /
156	# %begin
157	# < VarName >
158	# < < '=' \| '+=' > < dot* > > ?
159	# %end
160	# /
161	# Note: must use < > for grouping because there is no non-capturing group.
162
163
164	def _SplitAssignArg(arg, blame_word):
165	# type: (str, CompoundWord) -> AssignArg
166	"""Dynamically parse argument to declare, export, etc.
167
168	This is a fallback to the static parsing done below.
169	"""
170	# Note: it would be better to cache regcomp(), but we don't have an API for
171	# that, and it probably isn't a bottleneck now
172	m = util.simple_regex_search(ASSIGN_ARG_RE, arg)
173	if m is None:
174	e_die("Assignment builtin expected NAME=value, got %r" % arg,
175	blame_word)
176
177	var_name = m[1]
178	# m[2] is used for grouping; ERE doesn't have non-capturing groups
179
180	op = m[3]
181	if op is not None and len(op): # declare NAME=
182	val = value.Str(m[4]) # type: Optional[value_t]
183	append = op[0] == '+'
184	else: # declare NAME
185	val = None # no operator
186	append = False
187
188	return AssignArg(var_name, val, append, blame_word)
189
190
191	# NOTE: Could be done with util.BackslashEscape like glob_.GlobEscape().
192	def _BackslashEscape(s):
193	# type: (str) -> str
194	"""Double up backslashes.
195
196	Useful for strings about to be globbed and strings about to be IFS
197	escaped.
198	"""
199	return s.replace('\\', '\\\\')
200
201
202	def _ValueToPartValue(val, quoted, part_loc):
203	# type: (value_t, bool, word_part_t) -> part_value_t
204	"""Helper for VarSub evaluation.
205
206	Called by _EvalBracedVarSub and _EvalWordPart for SimpleVarSub.
207	"""
208	UP_val = val
209
210	with tagswitch(val) as case:
211	if case(value_e.Undef):
212	# This happens in the case of ${undef+foo}. We skipped _EmptyStrOrError,
213	# but we have to append to the empty string.
214	return part_value.String('', quoted, not quoted)
215
216	elif case(value_e.Str):
217	val = cast(value.Str, UP_val)
218	return part_value.String(val.s, quoted, not quoted)
219
220	elif case(value_e.BashArray):
221	val = cast(value.BashArray, UP_val)
222	return part_value.Array(val.strs)
223
224	elif case(value_e.BashAssoc):
225	val = cast(value.BashAssoc, UP_val)
226	# bash behavior: splice values!
227	return part_value.Array(val.d.values())
228
229	# Cases added for YSH
230	# value_e.List is also here - we use val_ops.stringify()s err message
231	elif case(value_e.Null, value_e.Bool, value_e.Int, value_e.Float,
232	value_e.Eggex, value_e.List):
233	s = val_ops.Stringify(val, loc.Missing)
234	return part_value.String(s, quoted, not quoted)
235
236	else:
237	raise error.TypeErr(val, "Can't substitute into word",
238	loc.WordPart(part_loc))
239
240	raise AssertionError('for -Wreturn-type in C++')
241
242
243	def _MakeWordFrames(part_vals):
244	# type: (List[part_value_t]) -> List[List[Tuple[str, bool, bool]]]
245	"""A word evaluates to a flat list of part_value (String or Array). frame
246	is a portion that results in zero or more args. It can never be joined.
247	This idea exists because of arrays like "$@" and "${a[@]}".
248
249	Example:
250
251	a=(1 '2 3' 4)
252	x=x
253	y=y
254
255	# This word
256	$x"${a[@]}"$y
257
258	# Results in Three frames:
259	[ ('x', False, True), ('1', True, False) ]
260	[ ('2 3', True, False) ]
261	[ ('4', True, False), ('y', False, True) ]
262
263	Note: A frame is a 3-tuple that's identical to part_value.String()? Maybe we
264	should make that top level type.
265	"""
266	current = [] # type: List[Tuple[str, bool, bool]]
267	frames = [current]
268
269	for p in part_vals:
270	UP_p = p
271
272	with tagswitch(p) as case:
273	if case(part_value_e.String):
274	p = cast(part_value.String, UP_p)
275	current.append((p.s, p.quoted, p.do_split))
276
277	elif case(part_value_e.Array):
278	p = cast(part_value.Array, UP_p)
279
280	is_first = True
281	for s in p.strs:
282	if s is None:
283	continue # ignore undefined array entries
284
285	# Arrays parts are always quoted; otherwise they would have decayed to
286	# a string.
287	portion = (s, True, False)
288	if is_first:
289	current.append(portion)
290	is_first = False
291	else:
292	current = [portion]
293	frames.append(current) # singleton frame
294
295	else:
296	raise AssertionError()
297
298	return frames
299
300
301	# TODO: This could be _MakeWordFrames and then sep.join(). It's redundant.
302	def _DecayPartValuesToString(part_vals, join_char):
303	# type: (List[part_value_t], str) -> str
304	# Decay ${a=x"$@"x} to string.
305	out = [] # type: List[str]
306	for p in part_vals:
307	UP_p = p
308	with tagswitch(p) as case:
309	if case(part_value_e.String):
310	p = cast(part_value.String, UP_p)
311	out.append(p.s)
312	elif case(part_value_e.Array):
313	p = cast(part_value.Array, UP_p)
314	# TODO: Eliminate double join for speed?
315	tmp = [s for s in p.strs if s is not None]
316	out.append(join_char.join(tmp))
317	else:
318	raise AssertionError()
319	return ''.join(out)
320
321
322	def _PerformSlice(
323	val, # type: value_t
324	begin, # type: int
325	length, # type: int
326	has_length, # type: bool
327	part, # type: BracedVarSub
328	arg0_val, # type: value.Str
329	):
330	# type: (...) -> value_t
331	UP_val = val
332	with tagswitch(val) as case:
333	if case(value_e.Str): # Slice UTF-8 characters in a string.
334	val = cast(value.Str, UP_val)
335	s = val.s
336	n = len(s)
337
338	if begin < 0: # Compute offset with unicode
339	byte_begin = n
340	num_iters = -begin
341	for _ in xrange(num_iters):
342	byte_begin = string_ops.PreviousUtf8Char(s, byte_begin)
343	else:
344	byte_begin = string_ops.AdvanceUtf8Chars(s, begin, 0)
345
346	if has_length:
347	if length < 0: # Compute offset with unicode
348	# Confusing: this is a POSITION
349	byte_end = n
350	num_iters = -length
351	for _ in xrange(num_iters):
352	byte_end = string_ops.PreviousUtf8Char(s, byte_end)
353	else:
354	byte_end = string_ops.AdvanceUtf8Chars(
355	s, length, byte_begin)
356	else:
357	byte_end = len(s)
358
359	substr = s[byte_begin:byte_end]
360	result = value.Str(substr) # type: value_t
361
362	elif case(value_e.BashArray): # Slice array entries.
363	val = cast(value.BashArray, UP_val)
364	# NOTE: This error is ALWAYS fatal in bash. It's inconsistent with
365	# strings.
366	if has_length and length < 0:
367	e_die(
368	"The length index of a array slice can't be negative: %d" %
369	length, loc.WordPart(part))
370
371	# Quirk: "begin" for positional arguments ($@ and $*) counts $0.
372	if arg0_val is not None:
373	orig = [arg0_val.s]
374	orig.extend(val.strs)
375	else:
376	orig = val.strs
377
378	n = len(orig)
379	if begin < 0:
380	i = n + begin # ${@:-3} starts counts from the end
381	else:
382	i = begin
383	strs = [] # type: List[str]
384	count = 0
385	while i < n:
386	if has_length and count == length: # length could be 0
387	break
388	s = orig[i]
389	if s is not None: # Unset elements don't count towards the length
390	strs.append(s)
391	count += 1
392	i += 1
393
394	result = value.BashArray(strs)
395
396	elif case(value_e.BashAssoc):
397	e_die("Can't slice associative arrays", loc.WordPart(part))
398
399	else:
400	raise error.TypeErr(val, 'Slice op expected Str or BashArray',
401	loc.WordPart(part))
402
403	return result
404
405
406	class StringWordEvaluator(object):
407	"""Interface used by ArithEvaluator / BoolEvaluator"""
408
409	def __init__(self):
410	# type: () -> None
411	"""Empty constructor for mycpp."""
412	pass
413
414	def EvalWordToString(self, w, eval_flags=0):
415	# type: (word_t, int) -> value.Str
416	raise NotImplementedError()
417
418
419	def _GetDollarHyphen(exec_opts):
420	# type: (optview.Exec) -> str
421	chars = [] # type: List[str]
422	if exec_opts.interactive():
423	chars.append('i')
424
425	if exec_opts.errexit():
426	chars.append('e')
427	if exec_opts.noglob():
428	chars.append('f')
429	if exec_opts.noexec():
430	chars.append('n')
431	if exec_opts.nounset():
432	chars.append('u')
433	# NO letter for pipefail?
434	if exec_opts.xtrace():
435	chars.append('x')
436	if exec_opts.noclobber():
437	chars.append('C')
438
439	# bash has:
440	# - c for sh -c, i for sh -i (mksh also has this)
441	# - h for hashing (mksh also has this)
442	# - B for brace expansion
443	return ''.join(chars)
444
445
446	class TildeEvaluator(object):
447
448	def __init__(self, mem, exec_opts):
449	# type: (Mem, optview.Exec) -> None
450	self.mem = mem
451	self.exec_opts = exec_opts
452
453	def GetMyHomeDir(self):
454	# type: () -> Optional[str]
455	"""Consult $HOME first, and then make a libc call.
456
457	Important: the libc call can FAIL, which is why we prefer $HOME. See issue
458	#1578.
459	"""
460	# First look up the HOME var, then ask the OS. This is what bash does.
461	val = self.mem.GetValue('HOME')
462	UP_val = val
463	if val.tag() == value_e.Str:
464	val = cast(value.Str, UP_val)
465	return val.s
466	return pyos.GetMyHomeDir()
467
468	def Eval(self, part):
469	# type: (word_part.TildeSub) -> str
470	"""Evaluates ~ and ~user, given a Lit_TildeLike token."""
471
472	if part.user_name is None:
473	result = self.GetMyHomeDir()
474	else:
475	result = pyos.GetHomeDir(part.user_name)
476
477	if result is None:
478	if self.exec_opts.strict_tilde():
479	e_die("Error expanding tilde (e.g. invalid user)", part.left)
480	else:
481	# Return ~ or ~user literally
482	result = '~'
483	if part.user_name is not None:
484	result = result + part.user_name # mycpp doesn't have +=
485
486	return result
487
488
489	class AbstractWordEvaluator(StringWordEvaluator):
490	"""Abstract base class for word evaluators.
491
492	Public entry points:
493	EvalWordToString EvalForPlugin EvalRhsWord
494	EvalWordSequence EvalWordSequence2
495	"""
496
497	def __init__(
498	self,
499	mem, # type: state.Mem
500	exec_opts, # type: optview.Exec
501	mutable_opts, # type: state.MutableOpts
502	tilde_ev, # type: TildeEvaluator
503	splitter, # type: SplitContext
504	errfmt, # type: ui.ErrorFormatter
505	):
506	# type: (...) -> None
507	self.arith_ev = None # type: sh_expr_eval.ArithEvaluator
508	self.expr_ev = None # type: expr_eval.ExprEvaluator
509	self.prompt_ev = None # type: prompt.Evaluator
510
511	self.unsafe_arith = None # type: sh_expr_eval.UnsafeArith
512
513	self.tilde_ev = tilde_ev
514
515	self.mem = mem # for $HOME, $1, etc.
516	self.exec_opts = exec_opts # for nounset
517	self.mutable_opts = mutable_opts # for _allow_command_sub
518	self.splitter = splitter
519	self.errfmt = errfmt
520
521	self.globber = glob_.Globber(exec_opts)
522
523	def CheckCircularDeps(self):
524	# type: () -> None
525	raise NotImplementedError()
526
527	def _EvalCommandSub(self, cs_part, quoted):
528	# type: (CommandSub, bool) -> part_value_t
529	"""Abstract since it has a side effect."""
530	raise NotImplementedError()
531
532	def _EvalProcessSub(self, cs_part):
533	# type: (CommandSub) -> part_value_t
534	"""Abstract since it has a side effect."""
535	raise NotImplementedError()
536
537	def _EvalVarNum(self, var_num):
538	# type: (int) -> value_t
539	assert var_num >= 0
540	return self.mem.GetArgNum(var_num)
541
542	def _EvalSpecialVar(self, op_id, quoted, vsub_state):
543	# type: (int, bool, VarSubState) -> value_t
544	"""Evaluate $?
545
546	and so forth
547	"""
548	# $@ is special -- it need to know whether it is in a double quoted
549	# context.
550	#
551	# - If it's $@ in a double quoted context, return an ARRAY.
552	# - If it's $@ in a normal context, return a STRING, which then will be
553	# subject to splitting.
554
555	if op_id in (Id.VSub_At, Id.VSub_Star):
556	argv = self.mem.GetArgv()
557	val = value.BashArray(argv) # type: value_t
558	if op_id == Id.VSub_At:
559	# "$@" evaluates to an array, $@ should be decayed
560	vsub_state.join_array = not quoted
561	else: # $* "$*" are both decayed
562	vsub_state.join_array = True
563
564	elif op_id == Id.VSub_Hyphen:
565	val = value.Str(_GetDollarHyphen(self.exec_opts))
566
567	else:
568	val = self.mem.GetSpecialVar(op_id)
569
570	return val
571
572	def _ApplyTestOp(
573	self,
574	val, # type: value_t
575	op, # type: suffix_op.Unary
576	quoted, # type: bool
577	part_vals, # type: Optional[List[part_value_t]]
578	vtest_place, # type: VTestPlace
579	blame_token, # type: Token
580	):
581	# type: (...) -> bool
582	"""
583	Returns:
584	Whether part_vals was mutated
585
586	${a:-} returns part_value[]
587	${a:+} returns part_value[]
588	${a:?error} returns error word?
589	${a:=} returns part_value[] but also needs self.mem for side effects.
590
591	So I guess it should return part_value[], and then a flag for raising an
592	error, and then a flag for assigning it?
593	The original BracedVarSub will have the name.
594
595	Example of needing multiple part_value[]
596
597	echo X-${a:-'def'"ault"}-X
598
599	We return two part values from the BracedVarSub. Also consider:
600
601	echo ${a:-x"$@"x}
602	"""
603	eval_flags = IS_SUBST
604	if quoted:
605	eval_flags \|= QUOTED
606
607	tok = op.op
608	# NOTE: Splicing part_values is necessary because of code like
609	# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
610	# do_glob/do_elide setting.
611	UP_val = val
612	with tagswitch(val) as case:
613	if case(value_e.Undef):
614	is_falsey = True
615	elif case(value_e.Str):
616	val = cast(value.Str, UP_val)
617	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_ColonEquals,
618	Id.VTest_ColonQMark, Id.VTest_ColonPlus):
619	is_falsey = len(val.s) == 0
620	else:
621	is_falsey = False
622	elif case(value_e.BashArray):
623	val = cast(value.BashArray, UP_val)
624	is_falsey = len(val.strs) == 0
625	elif case(value_e.BashAssoc):
626	val = cast(value.BashAssoc, UP_val)
627	is_falsey = len(val.d) == 0
628	else:
629	# value.Eggex, etc. are all false
630	is_falsey = False
631
632	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_Hyphen):
633	if is_falsey:
634	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
635	return True
636	else:
637	return False
638
639	# Inverse of the above.
640	elif tok.id in (Id.VTest_ColonPlus, Id.VTest_Plus):
641	if is_falsey:
642	return False
643	else:
644	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
645	return True
646
647	# Splice and assign
648	elif tok.id in (Id.VTest_ColonEquals, Id.VTest_Equals):
649	if is_falsey:
650	# Collect new part vals.
651	assign_part_vals = [] # type: List[part_value_t]
652	self._EvalRhsWordToParts(op.arg_word, assign_part_vals,
653	eval_flags)
654	# Append them to out param AND return them.
655	part_vals.extend(assign_part_vals)
656
657	if vtest_place.name is None:
658	# TODO: error context
659	e_die("Can't assign to special variable")
660	else:
661	# NOTE: This decays arrays too! 'shopt -s strict_array' could
662	# avoid it.
663	rhs_str = _DecayPartValuesToString(
664	assign_part_vals, self.splitter.GetJoinChar())
665	if vtest_place.index is None: # using None when no index
666	lval = location.LName(
667	vtest_place.name) # type: sh_lvalue_t
668	else:
669	var_name = vtest_place.name
670	var_index = vtest_place.index
671	UP_var_index = var_index
672
673	with tagswitch(var_index) as case:
674	if case(a_index_e.Int):
675	var_index = cast(a_index.Int, UP_var_index)
676	lval = sh_lvalue.Indexed(
677	var_name, var_index.i, loc.Missing)
678	elif case(a_index_e.Str):
679	var_index = cast(a_index.Str, UP_var_index)
680	lval = sh_lvalue.Keyed(var_name, var_index.s,
681	loc.Missing)
682	else:
683	raise AssertionError()
684
685	state.OshLanguageSetValue(self.mem, lval,
686	value.Str(rhs_str))
687	return True
688
689	else:
690	return False
691
692	elif tok.id in (Id.VTest_ColonQMark, Id.VTest_QMark):
693	if is_falsey:
694	# The arg is the error message
695	error_part_vals = [] # type: List[part_value_t]
696	self._EvalRhsWordToParts(op.arg_word, error_part_vals,
697	eval_flags)
698	error_str = _DecayPartValuesToString(
699	error_part_vals, self.splitter.GetJoinChar())
700	e_die("unset variable %r" % error_str, blame_token)
701
702	else:
703	return False
704
705	else:
706	raise AssertionError(tok.id)
707
708	def _Length(self, val, token):
709	# type: (value_t, Token) -> value_t
710	"""Returns the length of the value, for ${#var}"""
711	UP_val = val
712	with tagswitch(val) as case:
713	if case(value_e.Str):
714	val = cast(value.Str, UP_val)
715	# NOTE: Whether bash counts bytes or chars is affected by LANG
716	# environment variables.
717	# Should we respect that, or another way to select? set -o
718	# count-bytes?
719
720	# https://stackoverflow.com/questions/17368067/length-of-string-in-bash
721	try:
722	length = string_ops.CountUtf8Chars(val.s)
723	except error.Strict as e:
724	# Add this here so we don't have to add it so far down the stack.
725	# TODO: It's better to show BOTH this CODE an the actual DATA
726	# somehow.
727	e.location = token
728
729	if self.exec_opts.strict_word_eval():
730	raise
731	else:
732	# NOTE: Doesn't make the command exit with 1; it just returns a
733	# length of -1.
734	self.errfmt.PrettyPrintError(e, prefix='warning: ')
735	return value.Str('-1')
736
737	elif case(value_e.BashArray):
738	val = cast(value.BashArray, UP_val)
739	# There can be empty placeholder values in the array.
740	length = 0
741	for s in val.strs:
742	if s is not None:
743	length += 1
744
745	elif case(value_e.BashAssoc):
746	val = cast(value.BashAssoc, UP_val)
747	length = len(val.d)
748
749	else:
750	raise error.TypeErr(
751	val, "Length op expected Str, BashArray, BashAssoc", token)
752
753	return value.Str(str(length))
754
755	def _Keys(self, val, token):
756	# type: (value_t, Token) -> value_t
757	"""Return keys of a container, for ${!array[@]}"""
758
759	UP_val = val
760	with tagswitch(val) as case:
761	if case(value_e.BashArray):
762	val = cast(value.BashArray, UP_val)
763	# translation issue: tuple indices not supported in list comprehensions
764	#indices = [str(i) for i, s in enumerate(val.strs) if s is not None]
765	indices = [] # type: List[str]
766	for i, s in enumerate(val.strs):
767	if s is not None:
768	indices.append(str(i))
769	return value.BashArray(indices)
770
771	elif case(value_e.BashAssoc):
772	val = cast(value.BashAssoc, UP_val)
773	assert val.d is not None # for MyPy, so it's not Optional[]
774
775	# BUG: Keys aren't ordered according to insertion!
776	return value.BashArray(val.d.keys())
777
778	else:
779	raise error.TypeErr(val, 'Keys op expected Str', token)
780
781	def _EvalVarRef(self, val, blame_tok, quoted, vsub_state, vtest_place):
782	# type: (value_t, Token, bool, VarSubState, VTestPlace) -> value_t
783	"""Handles indirect expansion like ${!var} and ${!a[0]}.
784
785	Args:
786	blame_tok: 'foo' for ${!foo}
787	"""
788	UP_val = val
789	with tagswitch(val) as case:
790	if case(value_e.Undef):
791	return value.Undef # ${!undef} is just weird bash behavior
792
793	elif case(value_e.Str):
794	val = cast(value.Str, UP_val)
795	bvs_part = self.unsafe_arith.ParseVarRef(val.s, blame_tok)
796	return self._VarRefValue(bvs_part, quoted, vsub_state,
797	vtest_place)
798
799	elif case(value_e.BashArray): # caught earlier but OK
800	e_die('Indirect expansion of array')
801
802	elif case(value_e.BashAssoc): # caught earlier but OK
803	e_die('Indirect expansion of assoc array')
804
805	else:
806	raise error.TypeErr(val, 'Var Ref op expected Str', blame_tok)
807
808	def _ApplyUnarySuffixOp(self, val, op):
809	# type: (value_t, suffix_op.Unary) -> value_t
810	assert val.tag() != value_e.Undef
811
812	op_kind = consts.GetKind(op.op.id)
813
814	if op_kind == Kind.VOp1:
815	# NOTE: glob syntax is supported in ^ ^^ , ,, ! As well as % %% # ##.
816	# Detect has_extglob so that DoUnarySuffixOp doesn't use the fast
817	# shortcut for constant strings.
818	arg_val, has_extglob = self.EvalWordToPattern(op.arg_word)
819	assert arg_val.tag() == value_e.Str
820
821	UP_val = val
822	with tagswitch(val) as case:
823	if case(value_e.Str):
824	val = cast(value.Str, UP_val)
825	s = string_ops.DoUnarySuffixOp(val.s, op.op, arg_val.s,
826	has_extglob)
827	#log('%r %r -> %r', val.s, arg_val.s, s)
828	new_val = value.Str(s) # type: value_t
829
830	elif case(value_e.BashArray):
831	val = cast(value.BashArray, UP_val)
832	# ${a[@]#prefix} is VECTORIZED on arrays. YSH should have this too.
833	strs = [] # type: List[str]
834	for s in val.strs:
835	if s is not None:
836	strs.append(
837	string_ops.DoUnarySuffixOp(
838	s, op.op, arg_val.s, has_extglob))
839	new_val = value.BashArray(strs)
840
841	elif case(value_e.BashAssoc):
842	val = cast(value.BashAssoc, UP_val)
843	strs = []
844	for s in val.d.values():
845	strs.append(
846	string_ops.DoUnarySuffixOp(s, op.op, arg_val.s,
847	has_extglob))
848	new_val = value.BashArray(strs)
849
850	else:
851	raise error.TypeErr(
852	val, 'Unary op expected Str, BashArray, BashAssoc',
853	op.op)
854
855	else:
856	raise AssertionError(Kind_str(op_kind))
857
858	return new_val
859
860	def _PatSub(self, val, op):
861	# type: (value_t, suffix_op.PatSub) -> value_t
862
863	pat_val, has_extglob = self.EvalWordToPattern(op.pat)
864	# Extended globs aren't supported because we only translate * ? etc. to
865	# ERE. I don't think there's a straightforward translation from !(*.py) to
866	# ERE! You would need an engine that supports negation? (Derivatives?)
867	if has_extglob:
868	e_die('extended globs not supported in ${x//GLOB/}', op.pat)
869
870	if op.replace:
871	replace_val = self.EvalRhsWord(op.replace)
872	# Can't have an array, so must be a string
873	assert replace_val.tag() == value_e.Str, replace_val
874	replace_str = cast(value.Str, replace_val).s
875	else:
876	replace_str = ''
877
878	# note: doesn't support self.exec_opts.extglob()!
879	regex, warnings = glob_.GlobToERE(pat_val.s)
880	if len(warnings):
881	# TODO:
882	# - Add 'shopt -s strict_glob' mode and expose warnings.
883	# "Glob is not in CANONICAL FORM".
884	# - Propagate location info back to the 'op.pat' word.
885	pass
886	replacer = string_ops.GlobReplacer(regex, replace_str, op.slash_tok)
887
888	with tagswitch(val) as case2:
889	if case2(value_e.Str):
890	str_val = cast(value.Str, val)
891	s = replacer.Replace(str_val.s, op)
892	val = value.Str(s)
893
894	elif case2(value_e.BashArray):
895	array_val = cast(value.BashArray, val)
896	strs = [] # type: List[str]
897	for s in array_val.strs:
898	if s is not None:
899	strs.append(replacer.Replace(s, op))
900	val = value.BashArray(strs)
901
902	elif case2(value_e.BashAssoc):
903	assoc_val = cast(value.BashAssoc, val)
904	strs = []
905	for s in assoc_val.d.values():
906	strs.append(replacer.Replace(s, op))
907	val = value.BashArray(strs)
908
909	else:
910	raise error.TypeErr(
911	val, 'Pat Sub op expected Str, BashArray, BashAssoc',
912	op.slash_tok)
913
914	return val
915
916	def _Slice(self, val, op, var_name, part):
917	# type: (value_t, suffix_op.Slice, Optional[str], BracedVarSub) -> value_t
918
919	if op.begin:
920	begin = self.arith_ev.EvalToInt(op.begin)
921	else:
922	begin = 0
923
924	# Note: bash allows lengths to be negative (with odd semantics), but
925	# we don't allow that right now.
926	has_length = False
927	length = -1
928	if op.length:
929	has_length = True
930	length = self.arith_ev.EvalToInt(op.length)
931
932	try:
933	arg0_val = None # type: value.Str
934	if var_name is None: # $* or $@
935	arg0_val = self.mem.GetArg0()
936	val = _PerformSlice(val, begin, length, has_length, part, arg0_val)
937	except error.Strict as e:
938	if self.exec_opts.strict_word_eval():
939	raise
940	else:
941	self.errfmt.PrettyPrintError(e, prefix='warning: ')
942	with tagswitch(val) as case2:
943	if case2(value_e.Str):
944	val = value.Str('')
945	elif case2(value_e.BashArray):
946	val = value.BashArray([])
947	else:
948	raise NotImplementedError()
949	return val
950
951	def _Nullary(self, val, op, var_name):
952	# type: (value_t, Token, Optional[str]) -> Tuple[value.Str, bool]
953
954	UP_val = val
955	quoted2 = False
956	op_id = op.id
957	if op_id == Id.VOp0_P:
958	with tagswitch(val) as case:
959	if case(value_e.Str):
960	str_val = cast(value.Str, UP_val)
961	prompt = self.prompt_ev.EvalPrompt(str_val)
962	# readline gets rid of these, so we should too.
963	p = prompt.replace('\x01', '').replace('\x02', '')
964	result = value.Str(p)
965	else:
966	e_die("Can't use @P on %s" % ui.ValType(val), op)
967
968	elif op_id == Id.VOp0_Q:
969	with tagswitch(val) as case:
970	if case(value_e.Str):
971	str_val = cast(value.Str, UP_val)
972
973	# TODO: use fastfunc.ShellEncode or
974	# fastfunc.PosixShellEncode()
975	result = value.Str(j8_lite.MaybeShellEncode(str_val.s))
976	# oddly, 'echo ${x@Q}' is equivalent to 'echo "${x@Q}"' in bash
977	quoted2 = True
978	elif case(value_e.BashArray):
979	array_val = cast(value.BashArray, UP_val)
980
981	# TODO: should use fastfunc.ShellEncode
982	tmp = [j8_lite.MaybeShellEncode(s) for s in array_val.strs]
983	result = value.Str(' '.join(tmp))
984	else:
985	e_die("Can't use @Q on %s" % ui.ValType(val), op)
986
987	elif op_id == Id.VOp0_a:
988	# We're ONLY simluating -a and -A, not -r -x -n for now. See
989	# spec/ble-idioms.test.sh.
990	chars = [] # type: List[str]
991	with tagswitch(val) as case:
992	if case(value_e.BashArray):
993	chars.append('a')
994	elif case(value_e.BashAssoc):
995	chars.append('A')
996
997	if var_name is not None: # e.g. ${?@a} is allowed
998	cell = self.mem.GetCell(var_name)
999	if cell:
1000	if cell.readonly:
1001	chars.append('r')
1002	if cell.exported:
1003	chars.append('x')
1004	if cell.nameref:
1005	chars.append('n')
1006
1007	result = value.Str(''.join(chars))
1008
1009	else:
1010	e_die('Var op %r not implemented' % lexer.TokenVal(op), op)
1011
1012	return result, quoted2
1013
1014	def _WholeArray(self, val, part, quoted, vsub_state):
1015	# type: (value_t, BracedVarSub, bool, VarSubState) -> value_t
1016	op_id = cast(bracket_op.WholeArray, part.bracket_op).op_id
1017
1018	if op_id == Id.Lit_At:
1019	vsub_state.join_array = not quoted # ${a[@]} decays but "${a[@]}" doesn't
1020	UP_val = val
1021	with tagswitch(val) as case2:
1022	if case2(value_e.Undef):
1023	val = self._EmptyBashArrayOrError(part.token)
1024	elif case2(value_e.Str):
1025	val = cast(value.Str, UP_val)
1026	e_die("Can't index string with @", loc.WordPart(part))
1027	elif case2(value_e.BashArray):
1028	val = cast(value.BashArray, UP_val)
1029	# TODO: Is this a no-op? Just leave 'val' alone.
1030	val = value.BashArray(val.strs)
1031
1032	elif op_id == Id.Arith_Star:
1033	vsub_state.join_array = True # both ${a[]} and "${a[]}" decay
1034	UP_val = val
1035	with tagswitch(val) as case2:
1036	if case2(value_e.Undef):
1037	val = self._EmptyBashArrayOrError(part.token)
1038	elif case2(value_e.Str):
1039	val = cast(value.Str, UP_val)
1040	e_die("Can't index string with *", loc.WordPart(part))
1041	elif case2(value_e.BashArray):
1042	val = cast(value.BashArray, UP_val)
1043	# TODO: Is this a no-op? Just leave 'val' alone.
1044	# ${a[]} or "${a[]}" : vsub_state.join_array is always true
1045	val = value.BashArray(val.strs)
1046
1047	else:
1048	raise AssertionError(op_id) # unknown
1049
1050	return val
1051
1052	def _ArrayIndex(self, val, part, vtest_place):
1053	# type: (value_t, BracedVarSub, VTestPlace) -> value_t
1054	"""Process a numeric array index like ${a[i+1]}"""
1055	anode = cast(bracket_op.ArrayIndex, part.bracket_op).expr
1056
1057	UP_val = val
1058	with tagswitch(val) as case2:
1059	if case2(value_e.Undef):
1060	pass # it will be checked later
1061
1062	elif case2(value_e.Str):
1063	# Bash treats any string as an array, so we can't add our own
1064	# behavior here without making valid OSH invalid bash.
1065	e_die("Can't index string %r with integer" % part.var_name,
1066	part.token)
1067
1068	elif case2(value_e.BashArray):
1069	array_val = cast(value.BashArray, UP_val)
1070	index = self.arith_ev.EvalToInt(anode)
1071	vtest_place.index = a_index.Int(index)
1072
1073	s = GetArrayItem(array_val.strs, index)
1074
1075	if s is None:
1076	val = value.Undef
1077	else:
1078	val = value.Str(s)
1079
1080	elif case2(value_e.BashAssoc):
1081	assoc_val = cast(value.BashAssoc, UP_val)
1082	key = self.arith_ev.EvalWordToString(anode)
1083	vtest_place.index = a_index.Str(key) # out param
1084	s = assoc_val.d.get(key)
1085
1086	if s is None:
1087	val = value.Undef
1088	else:
1089	val = value.Str(s)
1090
1091	else:
1092	raise error.TypeErr(val,
1093	'Index op expected BashArray, BashAssoc',
1094	loc.WordPart(part))
1095
1096	return val
1097
1098	def _EvalDoubleQuoted(self, parts, part_vals):
1099	# type: (List[word_part_t], List[part_value_t]) -> None
1100	"""Evaluate parts of a DoubleQuoted part.
1101
1102	Args:
1103	part_vals: output param to append to.
1104	"""
1105	# Example of returning array:
1106	# $ a=(1 2); b=(3); $ c=(4 5)
1107	# $ argv "${a[@]}${b[@]}${c[@]}"
1108	# ['1', '234', '5']
1109	#
1110	# Example of multiple parts
1111	# $ argv "${a[@]}${undef[@]:-${c[@]}}"
1112	# ['1', '24', '5']
1113
1114	# Special case for "". The parser outputs (DoubleQuoted []), instead
1115	# of (DoubleQuoted [Literal '']). This is better but it means we
1116	# have to check for it.
1117	if len(parts) == 0:
1118	v = part_value.String('', True, False)
1119	part_vals.append(v)
1120	return
1121
1122	for p in parts:
1123	self._EvalWordPart(p, part_vals, QUOTED)
1124
1125	def EvalDoubleQuotedToString(self, dq_part):
1126	# type: (DoubleQuoted) -> str
1127	"""For double quoted strings in YSH expressions.
1128
1129	Example: var x = "$foo-${foo}"
1130	"""
1131	part_vals = [] # type: List[part_value_t]
1132	self._EvalDoubleQuoted(dq_part.parts, part_vals)
1133	return self._ConcatPartVals(part_vals, dq_part.left)
1134
1135	def _DecayArray(self, val):
1136	# type: (value.BashArray) -> value.Str
1137	"""Decay $* to a string."""
1138	assert val.tag() == value_e.BashArray, val
1139	sep = self.splitter.GetJoinChar()
1140	tmp = [s for s in val.strs if s is not None]
1141	return value.Str(sep.join(tmp))
1142
1143	def _EmptyStrOrError(self, val, token):
1144	# type: (value_t, Token) -> value_t
1145	if val.tag() != value_e.Undef:
1146	return val
1147
1148	if not self.exec_opts.nounset():
1149	return value.Str('')
1150
1151	tok_str = lexer.TokenVal(token)
1152	name = tok_str[1:] if tok_str.startswith('$') else tok_str
1153	e_die('Undefined variable %r' % name, token)
1154
1155	def _EmptyBashArrayOrError(self, token):
1156	# type: (Token) -> value_t
1157	assert token is not None
1158	if self.exec_opts.nounset():
1159	e_die('Undefined array %r' % lexer.TokenVal(token), token)
1160	else:
1161	return value.BashArray([])
1162
1163	def _EvalBracketOp(self, val, part, quoted, vsub_state, vtest_place):
1164	# type: (value_t, BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1165
1166	if part.bracket_op:
1167	with tagswitch(part.bracket_op) as case:
1168	if case(bracket_op_e.WholeArray):
1169	val = self._WholeArray(val, part, quoted, vsub_state)
1170
1171	elif case(bracket_op_e.ArrayIndex):
1172	val = self._ArrayIndex(val, part, vtest_place)
1173
1174	else:
1175	raise AssertionError(part.bracket_op.tag())
1176
1177	else: # no bracket op
1178	var_name = vtest_place.name
1179	if (var_name is not None and
1180	val.tag() in (value_e.BashArray, value_e.BashAssoc) and
1181	not vsub_state.is_type_query):
1182	if ShouldArrayDecay(var_name, self.exec_opts,
1183	not (part.prefix_op or part.suffix_op)):
1184	# for ${BASH_SOURCE}, etc.
1185	val = DecayArray(val)
1186	else:
1187	e_die(
1188	"Array %r can't be referred to as a scalar (without @ or *)"
1189	% var_name, loc.WordPart(part))
1190
1191	return val
1192
1193	def _VarRefValue(self, part, quoted, vsub_state, vtest_place):
1194	# type: (BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1195	"""Duplicates some logic from _EvalBracedVarSub, but returns a
1196	value_t."""
1197
1198	# 1. Evaluate from (var_name, var_num, token Id) -> value
1199	if part.token.id == Id.VSub_Name:
1200	vtest_place.name = part.var_name
1201	val = self.mem.GetValue(part.var_name)
1202
1203	elif part.token.id == Id.VSub_Number:
1204	var_num = int(part.var_name)
1205	val = self._EvalVarNum(var_num)
1206
1207	else:
1208	# $* decays
1209	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1210
1211	# We don't need var_index because it's only for L-Values of test ops?
1212	if self.exec_opts.eval_unsafe_arith():
1213	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1214	vtest_place)
1215	else:
1216	with state.ctx_Option(self.mutable_opts,
1217	[option_i._allow_command_sub], False):
1218	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1219	vtest_place)
1220
1221	return val
1222
1223	def _EvalBracedVarSub(self, part, part_vals, quoted):
1224	# type: (BracedVarSub, List[part_value_t], bool) -> None
1225	"""
1226	Args:
1227	part_vals: output param to append to.
1228	"""
1229	# We have different operators that interact in a non-obvious order.
1230	#
1231	# 1. bracket_op: value -> value, with side effect on vsub_state
1232	#
1233	# 2. prefix_op
1234	# a. length ${#x}: value -> value
1235	# b. var ref ${!ref}: can expand to an array
1236	#
1237	# 3. suffix_op:
1238	# a. no operator: you have a value
1239	# b. Test: value -> part_value[]
1240	# c. Other Suffix: value -> value
1241	#
1242	# 4. Process vsub_state.join_array here before returning.
1243	#
1244	# These cases are hard to distinguish:
1245	# - ${!prefix@} prefix query
1246	# - ${!array[@]} keys
1247	# - ${!ref} named reference
1248	# - ${!ref[0]} named reference
1249	#
1250	# I think we need several stages:
1251	#
1252	# 1. value: name, number, special, prefix query
1253	# 2. bracket_op
1254	# 3. prefix length -- this is TERMINAL
1255	# 4. indirection? Only for some of the ! cases
1256	# 5. string transformation suffix ops like ##
1257	# 6. test op
1258	# 7. vsub_state.join_array
1259
1260	# vsub_state.join_array is for joining "${a[*]}" and unquoted ${a[@]} AFTER
1261	# suffix ops are applied. If we take the length with a prefix op, the
1262	# distinction is ignored.
1263
1264	var_name = None # type: Optional[str] # used throughout the function
1265	vtest_place = VTestPlace(var_name, None) # For ${foo=default}
1266	vsub_state = VarSubState.CreateNull() # for $, ${a[]}, etc.
1267
1268	# 1. Evaluate from (var_name, var_num, token Id) -> value
1269	if part.token.id == Id.VSub_Name:
1270	# Handle ${!prefix@} first, since that looks at names and not values
1271	# Do NOT handle ${!A[@]@a} here!
1272	if (part.prefix_op is not None and part.bracket_op is None and
1273	part.suffix_op is not None and
1274	part.suffix_op.tag() == suffix_op_e.Nullary):
1275	nullary_op = cast(Token, part.suffix_op)
1276	# ${!x@} but not ${!x@P}
1277	if consts.GetKind(nullary_op.id) == Kind.VOp3:
1278	names = self.mem.VarNamesStartingWith(part.var_name)
1279	names.sort()
1280
1281	if quoted and nullary_op.id == Id.VOp3_At:
1282	part_vals.append(part_value.Array(names))
1283	else:
1284	sep = self.splitter.GetJoinChar()
1285	part_vals.append(
1286	part_value.String(sep.join(names), quoted, True))
1287	return # EARLY RETURN
1288
1289	var_name = part.var_name
1290	vtest_place.name = var_name # for _ApplyTestOp
1291
1292	val = self.mem.GetValue(var_name)
1293
1294	elif part.token.id == Id.VSub_Number:
1295	var_num = int(part.var_name)
1296	val = self._EvalVarNum(var_num)
1297	else:
1298	# $* decays
1299	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1300
1301	# Type query ${array@a} is a STRING, not an array
1302	# NOTE: ${array@Q} is ${array[0]@Q} in bash, which is different than
1303	# ${array[@]@Q}
1304	# TODO: An IR for ${} might simplify these lengthy conditions
1305	suffix_op_ = part.suffix_op
1306	if (suffix_op_ and suffix_op_.tag() == suffix_op_e.Nullary and
1307	cast(Token, suffix_op_).id == Id.VOp0_a):
1308	vsub_state.is_type_query = True
1309
1310	# 2. Bracket Op
1311	val = self._EvalBracketOp(val, part, quoted, vsub_state, vtest_place)
1312
1313	# Do the _EmptyStrOrError up front here, EXCEPT in the case of Kind.VTest
1314	suffix_is_test = False
1315	UP_op = suffix_op_
1316	if suffix_op_ is not None and suffix_op_.tag() == suffix_op_e.Unary:
1317	suffix_op_ = cast(suffix_op.Unary, UP_op)
1318	if consts.GetKind(suffix_op_.op.id) == Kind.VTest:
1319	suffix_is_test = True
1320
1321	if part.prefix_op:
1322	if part.prefix_op.id == Id.VSub_Pound: # ${#var} for length
1323	if not suffix_is_test: # undef -> '' BEFORE length
1324	val = self._EmptyStrOrError(val, part.token)
1325
1326	val = self._Length(val, part.token)
1327	# assume it's not quoted
1328	part_val = _ValueToPartValue(val, False, part)
1329	part_vals.append(part_val)
1330	return # EARLY EXIT: nothing else can come after length
1331
1332	elif part.prefix_op.id == Id.VSub_Bang:
1333	if part.bracket_op and part.bracket_op.tag(
1334	) == bracket_op_e.WholeArray:
1335	if suffix_is_test:
1336	# ${!a[@]-'default'} is a non-fatal runtime error in bash. Here
1337	# it's fatal.
1338	op_tok = cast(suffix_op.Unary, UP_op).op
1339	e_die('Test operation not allowed with ${!array[@]}',
1340	op_tok)
1341
1342	# ${!array[@]} to get indices/keys
1343	val = self._Keys(val, part.token)
1344	# already set vsub_State.join_array ABOVE
1345	else:
1346	# Process ${!ref}. SURPRISE: ${!a[0]} is an indirect expansion unlike
1347	# ${!a[@]} !
1348	# ${!ref} can expand into an array if ref='array[@]'
1349
1350	# Clear it now that we have a var ref
1351	vtest_place.name = None
1352	vtest_place.index = None
1353
1354	val = self._EvalVarRef(val, part.token, quoted, vsub_state,
1355	vtest_place)
1356
1357	if not suffix_is_test: # undef -> '' AFTER indirection
1358	val = self._EmptyStrOrError(val, part.token)
1359
1360	else:
1361	raise AssertionError(part.prefix_op)
1362
1363	else:
1364	if not suffix_is_test: # undef -> '' if no prefix op
1365	val = self._EmptyStrOrError(val, part.token)
1366
1367	quoted2 = False # another bit for @Q
1368	if suffix_op_:
1369	op = suffix_op_ # could get rid of this alias
1370
1371	with tagswitch(suffix_op_) as case:
1372	if case(suffix_op_e.Nullary):
1373	op = cast(Token, UP_op)
1374	val, quoted2 = self._Nullary(val, op, var_name)
1375
1376	elif case(suffix_op_e.Unary):
1377	op = cast(suffix_op.Unary, UP_op)
1378	if consts.GetKind(op.op.id) == Kind.VTest:
1379	if self._ApplyTestOp(val, op, quoted, part_vals,
1380	vtest_place, part.token):
1381	# e.g. to evaluate ${undef:-'default'}, we already appended
1382	# what we need
1383	return
1384
1385	else:
1386	# Other suffix: value -> value
1387	val = self._ApplyUnarySuffixOp(val, op)
1388
1389	elif case(suffix_op_e.PatSub): # PatSub, vectorized
1390	op = cast(suffix_op.PatSub, UP_op)
1391	val = self._PatSub(val, op)
1392
1393	elif case(suffix_op_e.Slice):
1394	op = cast(suffix_op.Slice, UP_op)
1395	val = self._Slice(val, op, var_name, part)
1396
1397	elif case(suffix_op_e.Static):
1398	op = cast(suffix_op.Static, UP_op)
1399	e_die('Not implemented', op.tok)
1400
1401	else:
1402	raise AssertionError()
1403
1404	# After applying suffixes, process join_array here.
1405	UP_val = val
1406	if val.tag() == value_e.BashArray:
1407	array_val = cast(value.BashArray, UP_val)
1408	if vsub_state.join_array:
1409	val = self._DecayArray(array_val)
1410	else:
1411	val = array_val
1412
1413	# For example, ${a} evaluates to value.Str(), but we want a
1414	# part_value.String().
1415	part_val = _ValueToPartValue(val, quoted or quoted2, part)
1416	part_vals.append(part_val)
1417
1418	def _ConcatPartVals(self, part_vals, location):
1419	# type: (List[part_value_t], loc_t) -> str
1420
1421	strs = [] # type: List[str]
1422	for part_val in part_vals:
1423	UP_part_val = part_val
1424	with tagswitch(part_val) as case:
1425	if case(part_value_e.String):
1426	part_val = cast(part_value.String, UP_part_val)
1427	s = part_val.s
1428
1429	elif case(part_value_e.Array):
1430	part_val = cast(part_value.Array, UP_part_val)
1431	if self.exec_opts.strict_array():
1432	# Examples: echo f > "$@"; local foo="$@"
1433	e_die("Illegal array word part (strict_array)",
1434	location)
1435	else:
1436	# It appears to not respect IFS
1437	# TODO: eliminate double join()?
1438	tmp = [s for s in part_val.strs if s is not None]
1439	s = ' '.join(tmp)
1440
1441	else:
1442	raise AssertionError()
1443
1444	strs.append(s)
1445
1446	return ''.join(strs)
1447
1448	def EvalBracedVarSubToString(self, part):
1449	# type: (BracedVarSub) -> str
1450	"""For double quoted strings in YSH expressions.
1451
1452	Example: var x = "$foo-${foo}"
1453	"""
1454	part_vals = [] # type: List[part_value_t]
1455	self._EvalBracedVarSub(part, part_vals, False)
1456	# blame ${ location
1457	return self._ConcatPartVals(part_vals, part.left)
1458
1459	def _EvalSimpleVarSub(self, part, part_vals, quoted):
1460	# type: (NameTok, List[part_value_t], bool) -> None
1461
1462	# TODO: use name
1463	token = part.left
1464	var_name = part.var_name
1465
1466	vsub_state = VarSubState.CreateNull()
1467
1468	# 1. Evaluate from (var_name, var_num, Token) -> defined, value
1469	if token.id == Id.VSub_DollarName:
1470	# TODO: Special case for LINENO
1471	val = self.mem.GetValue(var_name)
1472	if val.tag() in (value_e.BashArray, value_e.BashAssoc):
1473	if ShouldArrayDecay(var_name, self.exec_opts):
1474	# for $BASH_SOURCE, etc.
1475	val = DecayArray(val)
1476	else:
1477	e_die(
1478	"Array %r can't be referred to as a scalar (without @ or *)"
1479	% var_name, token)
1480
1481	elif token.id == Id.VSub_Number:
1482	var_num = int(var_name)
1483	val = self._EvalVarNum(var_num)
1484
1485	else:
1486	val = self._EvalSpecialVar(token.id, quoted, vsub_state)
1487
1488	#log('SIMPLE %s', part)
1489	val = self._EmptyStrOrError(val, token)
1490	UP_val = val
1491	if val.tag() == value_e.BashArray:
1492	array_val = cast(value.BashArray, UP_val)
1493	if vsub_state.join_array:
1494	val = self._DecayArray(array_val)
1495	else:
1496	val = array_val
1497
1498	v = _ValueToPartValue(val, quoted, part)
1499	part_vals.append(v)
1500
1501	def EvalSimpleVarSubToString(self, node):
1502	# type: (NameTok) -> str
1503	"""For double quoted strings in YSH expressions.
1504
1505	Example: var x = "$foo-${foo}"
1506	"""
1507	part_vals = [] # type: List[part_value_t]
1508	self._EvalSimpleVarSub(node, part_vals, False)
1509	return self._ConcatPartVals(part_vals, node.left)
1510
1511	def _EvalExtGlob(self, part, part_vals):
1512	# type: (word_part.ExtGlob, List[part_value_t]) -> None
1513	"""Evaluate @($x\|'foo'\|$(hostname)) and flatten it."""
1514	op = part.op
1515	if op.id == Id.ExtGlob_Comma:
1516	op_str = '@('
1517	else:
1518	op_str = lexer.TokenVal(op)
1519	# Do NOT split these.
1520	part_vals.append(part_value.String(op_str, False, False))
1521
1522	for i, w in enumerate(part.arms):
1523	if i != 0:
1524	part_vals.append(part_value.String('\|', False,
1525	False)) # separator
1526	# FLATTEN the tree of extglob "arms".
1527	self._EvalWordToParts(w, part_vals, EXTGLOB_NESTED)
1528	part_vals.append(part_value.String(')', False, False)) # closing )
1529
1530	def _TranslateExtGlob(self, part_vals, w, glob_parts, fnmatch_parts):
1531	# type: (List[part_value_t], CompoundWord, List[str], List[str]) -> None
1532	"""Translate a flattened WORD with an ExtGlob part to string patterns.
1533
1534	We need both glob and fnmatch patterns. _EvalExtGlob does the
1535	flattening.
1536	"""
1537	for i, part_val in enumerate(part_vals):
1538	UP_part_val = part_val
1539	with tagswitch(part_val) as case:
1540	if case(part_value_e.String):
1541	part_val = cast(part_value.String, UP_part_val)
1542	if part_val.quoted and not self.exec_opts.noglob():
1543	s = glob_.GlobEscape(part_val.s)
1544	else:
1545	# e.g. the @( and \| in @(foo\|bar) aren't quoted
1546	s = part_val.s
1547	glob_parts.append(s)
1548	fnmatch_parts.append(s) # from _EvalExtGlob()
1549
1550	elif case(part_value_e.Array):
1551	# Disallow array
1552	e_die(
1553	"Extended globs and arrays can't appear in the same word",
1554	w)
1555
1556	elif case(part_value_e.ExtGlob):
1557	part_val = cast(part_value.ExtGlob, UP_part_val)
1558	# keep appending fnmatch_parts, but repplace glob_parts with '*'
1559	self._TranslateExtGlob(part_val.part_vals, w, [],
1560	fnmatch_parts)
1561	glob_parts.append('*')
1562
1563	else:
1564	raise AssertionError()
1565
1566	def _EvalWordPart(self, part, part_vals, flags):
1567	# type: (word_part_t, List[part_value_t], int) -> None
1568	"""Evaluate a word part.
1569
1570	Called by _EvalWordToParts, EvalWordToString, and _EvalDoubleQuoted.
1571
1572	Args:
1573	part: What to evaluate
1574	part_vals: Output parameter.
1575	quoted: was the part quoted like "$x"
1576	is_subst: do_split
1577
1578	Returns:
1579	None
1580	"""
1581	quoted = bool(flags & QUOTED)
1582	is_subst = bool(flags & IS_SUBST)
1583
1584	UP_part = part
1585	with tagswitch(part) as case:
1586	if case(word_part_e.ShArrayLiteral):
1587	part = cast(ShArrayLiteral, UP_part)
1588	e_die("Unexpected array literal", loc.WordPart(part))
1589	elif case(word_part_e.BashAssocLiteral):
1590	part = cast(word_part.BashAssocLiteral, UP_part)
1591	e_die("Unexpected associative array literal",
1592	loc.WordPart(part))
1593
1594	elif case(word_part_e.Literal):
1595	part = cast(Token, UP_part)
1596	# Split if it's in a substitution.
1597	# That is: echo is not split, but ${foo:-echo} is split
1598	v = part_value.String(part.tval, quoted, is_subst)
1599	part_vals.append(v)
1600
1601	elif case(word_part_e.EscapedLiteral):
1602	part = cast(word_part.EscapedLiteral, UP_part)
1603	v = part_value.String(part.ch, True, False)
1604	part_vals.append(v)
1605
1606	elif case(word_part_e.SingleQuoted):
1607	part = cast(SingleQuoted, UP_part)
1608	s = word_compile.EvalSingleQuoted(part)
1609	v = part_value.String(s, True, False)
1610	part_vals.append(v)
1611
1612	elif case(word_part_e.DoubleQuoted):
1613	part = cast(DoubleQuoted, UP_part)
1614	self._EvalDoubleQuoted(part.parts, part_vals)
1615
1616	elif case(word_part_e.CommandSub):
1617	part = cast(CommandSub, UP_part)
1618	id_ = part.left_token.id
1619	if id_ in (Id.Left_DollarParen, Id.Left_AtParen,
1620	Id.Left_Backtick):
1621	sv = self._EvalCommandSub(part,
1622	quoted) # type: part_value_t
1623
1624	elif id_ in (Id.Left_ProcSubIn, Id.Left_ProcSubOut):
1625	sv = self._EvalProcessSub(part)
1626
1627	else:
1628	raise AssertionError(id_)
1629
1630	part_vals.append(sv)
1631
1632	elif case(word_part_e.SimpleVarSub):
1633	part = cast(NameTok, UP_part)
1634	self._EvalSimpleVarSub(part, part_vals, quoted)
1635
1636	elif case(word_part_e.BracedVarSub):
1637	part = cast(BracedVarSub, UP_part)
1638	self._EvalBracedVarSub(part, part_vals, quoted)
1639
1640	elif case(word_part_e.TildeSub):
1641	part = cast(word_part.TildeSub, UP_part)
1642	# We never parse a quoted string into a TildeSub.
1643	assert not quoted
1644	s = self.tilde_ev.Eval(part)
1645	v = part_value.String(s, True,
1646	False) # NOT split even when unquoted!
1647	part_vals.append(v)
1648
1649	elif case(word_part_e.ArithSub):
1650	part = cast(word_part.ArithSub, UP_part)
1651	num = self.arith_ev.EvalToBigInt(part.anode)
1652	v = part_value.String(mops.ToStr(num), quoted, not quoted)
1653	part_vals.append(v)
1654
1655	elif case(word_part_e.ExtGlob):
1656	part = cast(word_part.ExtGlob, UP_part)
1657	#if not self.exec_opts.extglob():
1658	# die() # disallow at runtime? Don't just decay
1659
1660	# Create a node to hold the flattened tree. The caller decides whether
1661	# to pass it to fnmatch() or replace it with '*' and pass it to glob().
1662	part_vals2 = [] # type: List[part_value_t]
1663	self._EvalExtGlob(part, part_vals2) # flattens tree
1664	part_vals.append(part_value.ExtGlob(part_vals2))
1665
1666	elif case(word_part_e.Splice):
1667	part = cast(word_part.Splice, UP_part)
1668	val = self.mem.GetValue(part.var_name)
1669
1670	strs = self.expr_ev.SpliceValue(val, part)
1671	part_vals.append(part_value.Array(strs))
1672
1673	elif case(word_part_e.ExprSub):
1674	part = cast(word_part.ExprSub, UP_part)
1675	part_val = self.expr_ev.EvalExprSub(part)
1676	part_vals.append(part_val)
1677
1678	else:
1679	raise AssertionError(part.tag())
1680
1681	def _EvalRhsWordToParts(self, w, part_vals, eval_flags=0):
1682	# type: (rhs_word_t, List[part_value_t], int) -> None
1683	quoted = bool(eval_flags & QUOTED)
1684
1685	UP_w = w
1686	with tagswitch(w) as case:
1687	if case(rhs_word_e.Empty):
1688	part_vals.append(part_value.String('', quoted, not quoted))
1689
1690	elif case(rhs_word_e.Compound):
1691	w = cast(CompoundWord, UP_w)
1692	self._EvalWordToParts(w, part_vals, eval_flags=eval_flags)
1693
1694	else:
1695	raise AssertionError()
1696
1697	def _EvalWordToParts(self, w, part_vals, eval_flags=0):
1698	# type: (CompoundWord, List[part_value_t], int) -> None
1699	"""Helper for EvalRhsWord, EvalWordSequence, etc.
1700
1701	Returns:
1702	Appends to part_vals. Note that this is a TREE.
1703	"""
1704	# Does the word have an extended glob? This is a special case because
1705	# of the way we use glob() and then fnmatch(..., FNM_EXTMATCH) to
1706	# implement extended globs. It's hard to carry that extra information
1707	# all the way past the word splitting stage.
1708
1709	# OSH semantic limitations: If a word has an extended glob part, then
1710	# 1. It can't have an array
1711	# 2. Word splitting of unquoted words isn't respected
1712
1713	word_part_vals = [] # type: List[part_value_t]
1714	has_extglob = False
1715	for p in w.parts:
1716	if p.tag() == word_part_e.ExtGlob:
1717	has_extglob = True
1718	self._EvalWordPart(p, word_part_vals, eval_flags)
1719
1720	# Caller REQUESTED extglob evaluation, AND we parsed word_part.ExtGlob()
1721	if has_extglob:
1722	if bool(eval_flags & EXTGLOB_FILES):
1723	# Treat the WHOLE word as a pattern. We need to TWO VARIANTS of the
1724	# word because of the way we use libc:
1725	# 1. With '*' for extglob parts
1726	# 2. With _EvalExtGlob() for extglob parts
1727
1728	glob_parts = [] # type: List[str]
1729	fnmatch_parts = [] # type: List[str]
1730	self._TranslateExtGlob(word_part_vals, w, glob_parts,
1731	fnmatch_parts)
1732
1733	#log('word_part_vals %s', word_part_vals)
1734	glob_pat = ''.join(glob_parts)
1735	fnmatch_pat = ''.join(fnmatch_parts)
1736	#log("glob %s fnmatch %s", glob_pat, fnmatch_pat)
1737
1738	results = [] # type: List[str]
1739	n = self.globber.ExpandExtended(glob_pat, fnmatch_pat, results)
1740	if n < 0:
1741	raise error.FailGlob(
1742	'Extended glob %r matched no files' % fnmatch_pat, w)
1743
1744	part_vals.append(part_value.Array(results))
1745	elif bool(eval_flags & EXTGLOB_NESTED):
1746	# We only glob at the TOP level of @(nested\|@(pattern))
1747	part_vals.extend(word_part_vals)
1748	else:
1749	# e.g. simple_word_eval, assignment builtin
1750	e_die('Extended glob not allowed in this word', w)
1751	else:
1752	part_vals.extend(word_part_vals)
1753
1754	def _PartValsToString(self, part_vals, w, eval_flags, strs):
1755	# type: (List[part_value_t], CompoundWord, int, List[str]) -> None
1756	"""Helper for EvalWordToString, similar to _ConcatPartVals() above.
1757
1758	Note: arg 'w' could just be a span ID
1759	"""
1760	for part_val in part_vals:
1761	UP_part_val = part_val
1762	with tagswitch(part_val) as case:
1763	if case(part_value_e.String):
1764	part_val = cast(part_value.String, UP_part_val)
1765	s = part_val.s
1766	if part_val.quoted:
1767	if eval_flags & QUOTE_FNMATCH:
1768	# [[ foo == /"".py ]] or case (.py) or ${x%.py} or ${x//*.py/}
1769	s = glob_.GlobEscape(s)
1770	elif eval_flags & QUOTE_ERE:
1771	s = glob_.ExtendedRegexEscape(s)
1772	strs.append(s)
1773
1774	elif case(part_value_e.Array):
1775	part_val = cast(part_value.Array, UP_part_val)
1776	if self.exec_opts.strict_array():
1777	# Examples: echo f > "$@"; local foo="$@"
1778
1779	# TODO: This attributes too coarsely, to the word rather than the
1780	# parts. Problem: the word is a TREE of parts, but we only have a
1781	# flat list of part_vals. The only case where we really get arrays
1782	# is "$@", "${a[@]}", "${a[@]//pat/replace}", etc.
1783	e_die(
1784	"This word should yield a string, but it contains an array",
1785	w)
1786
1787	# TODO: Maybe add detail like this.
1788	#e_die('RHS of assignment should only have strings. '
1789	# 'To assign arrays, use b=( "${a[@]}" )')
1790	else:
1791	# It appears to not respect IFS
1792	tmp = [s for s in part_val.strs if s is not None]
1793	s = ' '.join(tmp) # TODO: eliminate double join()?
1794	strs.append(s)
1795
1796	elif case(part_value_e.ExtGlob):
1797	part_val = cast(part_value.ExtGlob, UP_part_val)
1798
1799	# Extended globs are only allowed where we expect them!
1800	if not bool(eval_flags & QUOTE_FNMATCH):
1801	e_die('extended glob not allowed in this word', w)
1802
1803	# recursive call
1804	self._PartValsToString(part_val.part_vals, w, eval_flags,
1805	strs)
1806
1807	else:
1808	raise AssertionError()
1809
1810	def EvalWordToString(self, UP_w, eval_flags=0):
1811	# type: (word_t, int) -> value.Str
1812	"""Given a word, return a string.
1813
1814	Flags can contain a quoting algorithm.
1815	"""
1816	assert UP_w.tag() == word_e.Compound, UP_w
1817	w = cast(CompoundWord, UP_w)
1818
1819	if eval_flags == 0: # QUOTE_FNMATCH etc. breaks optimization
1820	fast_str = word_.FastStrEval(w)
1821	if fast_str is not None:
1822	return value.Str(fast_str)
1823
1824	# Could we additionally optimize a=$b, if we know $b isn't an array
1825	# etc.?
1826
1827	part_vals = [] # type: List[part_value_t]
1828	for p in w.parts:
1829	# this doesn't use eval_flags, which is slightly confusing
1830	self._EvalWordPart(p, part_vals, 0)
1831
1832	strs = [] # type: List[str]
1833	self._PartValsToString(part_vals, w, eval_flags, strs)
1834	return value.Str(''.join(strs))
1835
1836	def EvalWordToPattern(self, UP_w):
1837	# type: (rhs_word_t) -> Tuple[value.Str, bool]
1838	"""Like EvalWordToString, but returns whether we got ExtGlob."""
1839	if UP_w.tag() == rhs_word_e.Empty:
1840	return value.Str(''), False
1841
1842	assert UP_w.tag() == rhs_word_e.Compound, UP_w
1843	w = cast(CompoundWord, UP_w)
1844
1845	has_extglob = False
1846	part_vals = [] # type: List[part_value_t]
1847	for p in w.parts:
1848	# this doesn't use eval_flags, which is slightly confusing
1849	self._EvalWordPart(p, part_vals, 0)
1850	if p.tag() == word_part_e.ExtGlob:
1851	has_extglob = True
1852
1853	strs = [] # type: List[str]
1854	self._PartValsToString(part_vals, w, QUOTE_FNMATCH, strs)
1855	return value.Str(''.join(strs)), has_extglob
1856
1857	def EvalForPlugin(self, w):
1858	# type: (CompoundWord) -> value.Str
1859	"""Wrapper around EvalWordToString that prevents errors.
1860
1861	Runtime errors like $(( 1 / 0 )) and mutating $? like $(exit 42)
1862	are handled here.
1863
1864	Similar to ExprEvaluator.PluginCall().
1865	"""
1866	with state.ctx_Registers(self.mem): # to "sandbox" $? and $PIPESTATUS
1867	try:
1868	val = self.EvalWordToString(w)
1869	except error.FatalRuntime as e:
1870	val = value.Str('<Runtime error: %s>' % e.UserErrorString())
1871
1872	except (IOError, OSError) as e:
1873	val = value.Str('<I/O error: %s>' % pyutil.strerror(e))
1874
1875	except KeyboardInterrupt:
1876	val = value.Str('<Ctrl-C>')
1877
1878	return val
1879
1880	def EvalRhsWord(self, UP_w):
1881	# type: (rhs_word_t) -> value_t
1882	"""Used for RHS of assignment.
1883
1884	There is no splitting.
1885	"""
1886	if UP_w.tag() == rhs_word_e.Empty:
1887	return value.Str('')
1888
1889	assert UP_w.tag() == word_e.Compound, UP_w
1890	w = cast(CompoundWord, UP_w)
1891
1892	if len(w.parts) == 1:
1893	part0 = w.parts[0]
1894	UP_part0 = part0
1895	tag = part0.tag()
1896	# Special case for a=(1 2). ShArrayLiteral won't appear in words that
1897	# don't look like assignments.
1898	if tag == word_part_e.ShArrayLiteral:
1899	part0 = cast(ShArrayLiteral, UP_part0)
1900	array_words = part0.words
1901	words = braces.BraceExpandWords(array_words)
1902	strs = self.EvalWordSequence(words)
1903	return value.BashArray(strs)
1904
1905	if tag == word_part_e.BashAssocLiteral:
1906	part0 = cast(word_part.BashAssocLiteral, UP_part0)
1907	d = NewDict() # type: Dict[str, str]
1908	for pair in part0.pairs:
1909	k = self.EvalWordToString(pair.key)
1910	v = self.EvalWordToString(pair.value)
1911	d[k.s] = v.s
1912	return value.BashAssoc(d)
1913
1914	# If RHS doesn't look like a=( ... ), then it must be a string.
1915	return self.EvalWordToString(w)
1916
1917	def _EvalWordFrame(self, frame, argv):
1918	# type: (List[Tuple[str, bool, bool]], List[str]) -> None
1919	all_empty = True
1920	all_quoted = True
1921	any_quoted = False
1922
1923	#log('--- frame %s', frame)
1924
1925	for s, quoted, _ in frame:
1926	if len(s):
1927	all_empty = False
1928
1929	if quoted:
1930	any_quoted = True
1931	else:
1932	all_quoted = False
1933
1934	# Elision of ${empty}${empty} but not $empty"$empty" or $empty""
1935	if all_empty and not any_quoted:
1936	return
1937
1938	# If every frag is quoted, e.g. "$a$b" or any part in "${a[@]}"x, then
1939	# don't do word splitting or globbing.
1940	if all_quoted:
1941	tmp = [s for s, _, _ in frame]
1942	a = ''.join(tmp)
1943	argv.append(a)
1944	return
1945
1946	will_glob = not self.exec_opts.noglob()
1947
1948	# Array of strings, some of which are BOTH IFS-escaped and GLOB escaped!
1949	frags = [] # type: List[str]
1950	for frag, quoted, do_split in frame:
1951	if will_glob and quoted:
1952	frag = glob_.GlobEscape(frag)
1953	else:
1954	# If we have a literal \, then we turn it into \\\\.
1955	# Splitting takes \\\\ -> \\
1956	# Globbing takes \\ to \ if it doesn't match
1957	frag = _BackslashEscape(frag)
1958
1959	if do_split:
1960	frag = _BackslashEscape(frag)
1961	else:
1962	frag = self.splitter.Escape(frag)
1963
1964	frags.append(frag)
1965
1966	flat = ''.join(frags)
1967	#log('flat: %r', flat)
1968
1969	args = self.splitter.SplitForWordEval(flat)
1970
1971	# space=' '; argv $space"". We have a quoted part, but we CANNOT elide.
1972	# Add it back and don't bother globbing.
1973	if len(args) == 0 and any_quoted:
1974	argv.append('')
1975	return
1976
1977	#log('split args: %r', args)
1978	for a in args:
1979	if glob_.LooksLikeGlob(a):
1980	n = self.globber.Expand(a, argv)
1981	if n < 0:
1982	# TODO: location info, with span IDs carried through the frame
1983	raise error.FailGlob('Pattern %r matched no files' % a,
1984	loc.Missing)
1985	else:
1986	argv.append(glob_.GlobUnescape(a))
1987
1988	def _EvalWordToArgv(self, w):
1989	# type: (CompoundWord) -> List[str]
1990	"""Helper for _EvalAssignBuiltin.
1991
1992	Splitting and globbing are disabled for assignment builtins.
1993
1994	Example: declare -"${a[@]}" b=(1 2)
1995	where a is [x b=a d=a]
1996	"""
1997	part_vals = [] # type: List[part_value_t]
1998	self._EvalWordToParts(w, part_vals, 0) # not double quoted
1999	frames = _MakeWordFrames(part_vals)
2000	argv = [] # type: List[str]
2001	for frame in frames:
2002	if len(frame): # empty array gives empty frame!
2003	tmp = [s for (s, _, _) in frame]
2004	argv.append(''.join(tmp)) # no split or glob
2005	#log('argv: %s', argv)
2006	return argv
2007
2008	def _EvalAssignBuiltin(self, builtin_id, arg0, words):
2009	# type: (builtin_t, str, List[CompoundWord]) -> cmd_value.Assign
2010	"""Handles both static and dynamic assignment, e.g.
2011
2012	x='foo=bar' local a=(1 2) $x
2013	"""
2014	# Grammar:
2015	#
2016	# ('builtin' \| 'command')* keyword flag* pair*
2017	# flag = [-+].*
2018	#
2019	# There is also command -p, but we haven't implemented it. Maybe just punt
2020	# on it. Punted on 'builtin' and 'command' for now too.
2021
2022	eval_to_pairs = True # except for -f and -F
2023	started_pairs = False
2024
2025	flags = [arg0] # initial flags like -p, and -f -F name1 name2
2026	flag_locs = [words[0]]
2027	assign_args = [] # type: List[AssignArg]
2028
2029	n = len(words)
2030	for i in xrange(1, n): # skip first word
2031	w = words[i]
2032
2033	if word_.IsVarLike(w):
2034	started_pairs = True # Everything from now on is an assign_pair
2035
2036	if started_pairs:
2037	left_token, close_token, part_offset = word_.DetectShAssignment(
2038	w)
2039	if left_token: # Detected statically
2040	if left_token.id != Id.Lit_VarLike:
2041	# (not guaranteed since started_pairs is set twice)
2042	e_die('LHS array not allowed in assignment builtin', w)
2043
2044	if lexer.IsPlusEquals(left_token):
2045	var_name = lexer.TokenSliceRight(left_token, -2)
2046	append = True
2047	else:
2048	var_name = lexer.TokenSliceRight(left_token, -1)
2049	append = False
2050
2051	if part_offset == len(w.parts):
2052	rhs = rhs_word.Empty # type: rhs_word_t
2053	else:
2054	# tmp is for intersection of C++/MyPy type systems
2055	tmp = CompoundWord(w.parts[part_offset:])
2056	word_.TildeDetectAssign(tmp)
2057	rhs = tmp
2058
2059	with state.ctx_AssignBuiltin(self.mutable_opts):
2060	right = self.EvalRhsWord(rhs)
2061
2062	arg2 = AssignArg(var_name, right, append, w)
2063	assign_args.append(arg2)
2064
2065	else: # e.g. export $dynamic
2066	argv = self._EvalWordToArgv(w)
2067	for arg in argv:
2068	arg2 = _SplitAssignArg(arg, w)
2069	assign_args.append(arg2)
2070
2071	else:
2072	argv = self._EvalWordToArgv(w)
2073	for arg in argv:
2074	if arg.startswith('-') or arg.startswith('+'):
2075	# e.g. declare -r +r
2076	flags.append(arg)
2077	flag_locs.append(w)
2078
2079	# Shortcut that relies on -f and -F always meaning "function" for
2080	# all assignment builtins
2081	if 'f' in arg or 'F' in arg:
2082	eval_to_pairs = False
2083
2084	else: # e.g. export $dynamic
2085	if eval_to_pairs:
2086	arg2 = _SplitAssignArg(arg, w)
2087	assign_args.append(arg2)
2088	started_pairs = True
2089	else:
2090	flags.append(arg)
2091
2092	return cmd_value.Assign(builtin_id, flags, flag_locs, assign_args)
2093
2094	def SimpleEvalWordSequence2(self, words, allow_assign):
2095	# type: (List[CompoundWord], bool) -> cmd_value_t
2096	"""Simple word evaluation for YSH."""
2097	strs = [] # type: List[str]
2098	locs = [] # type: List[CompoundWord]
2099
2100	for i, w in enumerate(words):
2101	# No globbing in the first arg for command.Simple.
2102	if i == 0 and allow_assign:
2103	strs0 = self._EvalWordToArgv(w) # respects strict-array
2104	if len(strs0) == 1:
2105	arg0 = strs0[0]
2106	builtin_id = consts.LookupAssignBuiltin(arg0)
2107	if builtin_id != consts.NO_INDEX:
2108	# Same logic as legacy word eval, with no splitting
2109	return self._EvalAssignBuiltin(builtin_id, arg0, words)
2110
2111	strs.extend(strs0)
2112	for _ in strs0:
2113	locs.append(w)
2114	continue
2115
2116	if glob_.LooksLikeStaticGlob(w):
2117	val = self.EvalWordToString(w) # respects strict-array
2118	num_appended = self.globber.Expand(val.s, strs)
2119	if num_appended < 0:
2120	raise error.FailGlob('Pattern %r matched no files' % val.s,
2121	w)
2122	for _ in xrange(num_appended):
2123	locs.append(w)
2124	continue
2125
2126	part_vals = [] # type: List[part_value_t]
2127	self._EvalWordToParts(w, part_vals, 0) # not quoted
2128
2129	if 0:
2130	log('')
2131	log('Static: part_vals after _EvalWordToParts:')
2132	for entry in part_vals:
2133	log(' %s', entry)
2134
2135	# Still need to process
2136	frames = _MakeWordFrames(part_vals)
2137
2138	if 0:
2139	log('')
2140	log('Static: frames after _MakeWordFrames:')
2141	for entry in frames:
2142	log(' %s', entry)
2143
2144	# We will still allow x"${a[@]"x, though it's deprecated by @a, which
2145	# disallows such expressions at parse time.
2146	for frame in frames:
2147	if len(frame): # empty array gives empty frame!
2148	tmp = [s for (s, _, _) in frame]
2149	strs.append(''.join(tmp)) # no split or glob
2150	locs.append(w)
2151
2152	return cmd_value.Argv(strs, locs, None, None, None)
2153
2154	def EvalWordSequence2(self, words, allow_assign=False):
2155	# type: (List[CompoundWord], bool) -> cmd_value_t
2156	"""Turns a list of Words into a list of strings.
2157
2158	Unlike the EvalWord*() methods, it does globbing.
2159
2160	Args:
2161	words: list of Word instances
2162
2163	Returns:
2164	argv: list of string arguments, or None if there was an eval error
2165	"""
2166	if self.exec_opts.simple_word_eval():
2167	return self.SimpleEvalWordSequence2(words, allow_assign)
2168
2169	# Parse time:
2170	# 1. brace expansion. TODO: Do at parse time.
2171	# 2. Tilde detection. DONE at parse time. Only if Id.Lit_Tilde is the
2172	# first WordPart.
2173	#
2174	# Run time:
2175	# 3. tilde sub, var sub, command sub, arith sub. These are all
2176	# "concurrent" on WordParts. (optional process sub with <() )
2177	# 4. word splitting. Can turn this off with a shell option? Definitely
2178	# off for oil.
2179	# 5. globbing -- several exec_opts affect this: nullglob, safeglob, etc.
2180
2181	#log('W %s', words)
2182	strs = [] # type: List[str]
2183	locs = [] # type: List[CompoundWord]
2184
2185	n = 0
2186	for i, w in enumerate(words):
2187	fast_str = word_.FastStrEval(w)
2188	if fast_str is not None:
2189	strs.append(fast_str)
2190	locs.append(w)
2191
2192	# e.g. the 'local' in 'local a=b c=d' will be here
2193	if allow_assign and i == 0:
2194	builtin_id = consts.LookupAssignBuiltin(fast_str)
2195	if builtin_id != consts.NO_INDEX:
2196	return self._EvalAssignBuiltin(builtin_id, fast_str,
2197	words)
2198	continue
2199
2200	part_vals = [] # type: List[part_value_t]
2201	self._EvalWordToParts(w, part_vals, EXTGLOB_FILES)
2202
2203	# DYNAMICALLY detect if we're going to run an assignment builtin, and
2204	# change the rest of the evaluation algorithm if so.
2205	#
2206	# We want to allow:
2207	# e=export
2208	# $e foo=bar
2209	#
2210	# But we don't want to evaluate the first word twice in the case of:
2211	# $(some-command) --flag
2212	if allow_assign and i == 0 and len(part_vals) == 1:
2213	val0 = part_vals[0]
2214	UP_val0 = val0
2215	if val0.tag() == part_value_e.String:
2216	val0 = cast(part_value.String, UP_val0)
2217	if not val0.quoted:
2218	builtin_id = consts.LookupAssignBuiltin(val0.s)
2219	if builtin_id != consts.NO_INDEX:
2220	return self._EvalAssignBuiltin(
2221	builtin_id, val0.s, words)
2222
2223	if 0:
2224	log('')
2225	log('part_vals after _EvalWordToParts:')
2226	for entry in part_vals:
2227	log(' %s', entry)
2228
2229	frames = _MakeWordFrames(part_vals)
2230	if 0:
2231	log('')
2232	log('frames after _MakeWordFrames:')
2233	for entry in frames:
2234	log(' %s', entry)
2235
2236	# Do splitting and globbing. Each frame will append zero or more args.
2237	for frame in frames:
2238	self._EvalWordFrame(frame, strs)
2239
2240	# Fill in locations parallel to strs.
2241	n_next = len(strs)
2242	for _ in xrange(n_next - n):
2243	locs.append(w)
2244	n = n_next
2245
2246	# A non-assignment command.
2247	# NOTE: Can't look up builtins here like we did for assignment, because
2248	# functions can override builtins.
2249	return cmd_value.Argv(strs, locs, None, None, None)
2250
2251	def EvalWordSequence(self, words):
2252	# type: (List[CompoundWord]) -> List[str]
2253	"""For arrays and for loops.
2254
2255	They don't allow assignment builtins.
2256	"""
2257	UP_cmd_val = self.EvalWordSequence2(words)
2258
2259	assert UP_cmd_val.tag() == cmd_value_e.Argv
2260	cmd_val = cast(cmd_value.Argv, UP_cmd_val)
2261	return cmd_val.argv
2262
2263
2264	class NormalWordEvaluator(AbstractWordEvaluator):
2265
2266	def __init__(
2267	self,
2268	mem, # type: state.Mem
2269	exec_opts, # type: optview.Exec
2270	mutable_opts, # type: state.MutableOpts
2271	tilde_ev, # type: TildeEvaluator
2272	splitter, # type: SplitContext
2273	errfmt, # type: ErrorFormatter
2274	):
2275	# type: (...) -> None
2276	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2277	tilde_ev, splitter, errfmt)
2278	self.shell_ex = None # type: _Executor
2279
2280	def CheckCircularDeps(self):
2281	# type: () -> None
2282	assert self.arith_ev is not None
2283	# Disabled for pure OSH
2284	#assert self.expr_ev is not None
2285	assert self.shell_ex is not None
2286	assert self.prompt_ev is not None
2287
2288	def _EvalCommandSub(self, cs_part, quoted):
2289	# type: (CommandSub, bool) -> part_value_t
2290	stdout_str = self.shell_ex.RunCommandSub(cs_part)
2291	if cs_part.left_token.id == Id.Left_AtParen:
2292	strs = self.splitter.SplitForWordEval(stdout_str)
2293	return part_value.Array(strs)
2294	else:
2295	return part_value.String(stdout_str, quoted, not quoted)
2296
2297	def _EvalProcessSub(self, cs_part):
2298	# type: (CommandSub) -> part_value.String
2299	dev_path = self.shell_ex.RunProcessSub(cs_part)
2300	# pretend it's quoted; no split or glob
2301	return part_value.String(dev_path, True, False)
2302
2303
2304	_DUMMY = '__NO_COMMAND_SUB__'
2305
2306
2307	class CompletionWordEvaluator(AbstractWordEvaluator):
2308	"""An evaluator that has no access to an executor.
2309
2310	NOTE: core/completion.py doesn't actually try to use these strings to
2311	complete. If you have something like 'echo $(echo hi)/f<TAB>', it sees the
2312	inner command as the last one, and knows that it is not at the end of the
2313	line.
2314	"""
2315
2316	def __init__(
2317	self,
2318	mem, # type: state.Mem
2319	exec_opts, # type: optview.Exec
2320	mutable_opts, # type: state.MutableOpts
2321	tilde_ev, # type: TildeEvaluator
2322	splitter, # type: SplitContext
2323	errfmt, # type: ErrorFormatter
2324	):
2325	# type: (...) -> None
2326	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2327	tilde_ev, splitter, errfmt)
2328
2329	def CheckCircularDeps(self):
2330	# type: () -> None
2331	assert self.prompt_ev is not None
2332	assert self.arith_ev is not None
2333	assert self.expr_ev is not None
2334
2335	def _EvalCommandSub(self, cs_part, quoted):
2336	# type: (CommandSub, bool) -> part_value_t
2337	if cs_part.left_token.id == Id.Left_AtParen:
2338	return part_value.Array([_DUMMY])
2339	else:
2340	return part_value.String(_DUMMY, quoted, not quoted)
2341
2342	def _EvalProcessSub(self, cs_part):
2343	# type: (CommandSub) -> part_value.String
2344	# pretend it's quoted; no split or glob
2345	return part_value.String('__NO_PROCESS_SUB__', True, False)
2346
2347
2348	# vim: sw=4