osh/word_eval.py

OILS / osh / word_eval.py View on Github | oilshell.org

2375 lines, 1433 significant

1	"""
2	word_eval.py - Evaluator for the word language.
3	"""
4
5	from _devbuild.gen.id_kind_asdl import Id, Kind, Kind_str
6	from _devbuild.gen.syntax_asdl import (
7	Token,
8	SimpleVarSub,
9	loc,
10	loc_t,
11	BracedVarSub,
12	CommandSub,
13	bracket_op,
14	bracket_op_e,
15	suffix_op,
16	suffix_op_e,
17	ShArrayLiteral,
18	SingleQuoted,
19	DoubleQuoted,
20	word_e,
21	word_t,
22	CompoundWord,
23	rhs_word,
24	rhs_word_e,
25	rhs_word_t,
26	word_part,
27	word_part_e,
28	)
29	from _devbuild.gen.runtime_asdl import (
30	part_value,
31	part_value_e,
32	part_value_t,
33	cmd_value,
34	cmd_value_e,
35	cmd_value_t,
36	AssignArg,
37	a_index,
38	a_index_e,
39	VTestPlace,
40	VarSubState,
41	Piece,
42	)
43	from _devbuild.gen.option_asdl import option_i
44	from _devbuild.gen.value_asdl import (
45	value,
46	value_e,
47	value_t,
48	sh_lvalue,
49	sh_lvalue_t,
50	)
51	from core import error
52	from core import pyos
53	from core import pyutil
54	from core import state
55	from core import ui
56	from core import util
57	from data_lang import j8
58	from data_lang import j8_lite
59	from core.error import e_die
60	from frontend import consts
61	from frontend import lexer
62	from frontend import location
63	from mycpp import mops
64	from mycpp.mylib import log, tagswitch, NewDict
65	from osh import braces
66	from osh import glob_
67	from osh import string_ops
68	from osh import word_
69	from ysh import expr_eval
70	from ysh import val_ops
71
72	from typing import Optional, Tuple, List, Dict, cast, TYPE_CHECKING
73
74	if TYPE_CHECKING:
75	from _devbuild.gen.syntax_asdl import word_part_t
76	from _devbuild.gen.option_asdl import builtin_t
77	from core import optview
78	from core.state import Mem
79	from core.ui import ErrorFormatter
80	from core.vm import _Executor
81	from osh.split import SplitContext
82	from osh import prompt
83	from osh import sh_expr_eval
84
85	# Flags for _EvalWordToParts and _EvalWordPart (not all are used for both)
86	QUOTED = 1 << 0
87	IS_SUBST = 1 << 1
88
89	EXTGLOB_FILES = 1 << 2 # allow @(cc) from file system?
90	EXTGLOB_MATCH = 1 << 3 # allow @(cc) in pattern matching?
91	EXTGLOB_NESTED = 1 << 4 # for @(one\|!(two\|three))
92
93	# For EvalWordToString
94	QUOTE_FNMATCH = 1 << 5
95	QUOTE_ERE = 1 << 6
96
97	# For compatibility, ${BASH_SOURCE} and ${BASH_SOURCE[@]} are both valid.
98	# Ditto for ${FUNCNAME} and ${BASH_LINENO}.
99	_STRING_AND_ARRAY = ['BASH_SOURCE', 'FUNCNAME', 'BASH_LINENO']
100
101
102	def ShouldArrayDecay(var_name, exec_opts, is_plain_var_sub=True):
103	# type: (str, optview.Exec, bool) -> bool
104	"""Return whether we should allow ${a} to mean ${a[0]}."""
105	return (not exec_opts.strict_array() or
106	is_plain_var_sub and var_name in _STRING_AND_ARRAY)
107
108
109	def DecayArray(val):
110	# type: (value_t) -> value_t
111	"""Resolve ${array} to ${array[0]}."""
112	if val.tag() == value_e.BashArray:
113	array_val = cast(value.BashArray, val)
114	s = array_val.strs[0] if len(array_val.strs) else None
115	elif val.tag() == value_e.BashAssoc:
116	assoc_val = cast(value.BashAssoc, val)
117	s = assoc_val.d['0'] if '0' in assoc_val.d else None
118	else:
119	raise AssertionError(val.tag())
120
121	if s is None:
122	return value.Undef
123	else:
124	return value.Str(s)
125
126
127	def GetArrayItem(strs, index):
128	# type: (List[str], int) -> Optional[str]
129
130	n = len(strs)
131	if index < 0:
132	index += n
133
134	if 0 <= index and index < n:
135	# TODO: strs->index() has a redundant check for (i < 0)
136	s = strs[index]
137	# note: s could be None because representation is sparse
138	else:
139	s = None
140	return s
141
142
143	# Use libc to parse NAME, NAME=value, and NAME+=value. We want submatch
144	# extraction, but I haven't used that in re2c, and we would need a new kind of
145	# binding.
146	#
147	ASSIGN_ARG_RE = '^([a-zA-Z_][a-zA-Z0-9_])((=\|\+=)(.))?$'
148
149	# Eggex equivalent:
150	#
151	# VarName = /
152	# [a-z A-Z _ ]
153	# [a-z A-Z 0-9 _ ]*
154	# /
155	#
156	# SplitArg = /
157	# %begin
158	# < VarName >
159	# < < '=' \| '+=' > < dot* > > ?
160	# %end
161	# /
162	# Note: must use < > for grouping because there is no non-capturing group.
163
164
165	def _SplitAssignArg(arg, blame_word):
166	# type: (str, CompoundWord) -> AssignArg
167	"""Dynamically parse argument to declare, export, etc.
168
169	This is a fallback to the static parsing done below.
170	"""
171	# Note: it would be better to cache regcomp(), but we don't have an API for
172	# that, and it probably isn't a bottleneck now
173	m = util.RegexSearch(ASSIGN_ARG_RE, arg)
174	if m is None:
175	e_die("Assignment builtin expected NAME=value, got %r" % arg,
176	blame_word)
177
178	var_name = m[1]
179	# m[2] is used for grouping; ERE doesn't have non-capturing groups
180
181	op = m[3]
182	assert op is not None, op
183	if len(op): # declare NAME=
184	val = value.Str(m[4]) # type: Optional[value_t]
185	append = op[0] == '+'
186	else: # declare NAME
187	val = None # no operator
188	append = False
189
190	return AssignArg(var_name, val, append, blame_word)
191
192
193	# NOTE: Could be done with util.BackslashEscape like glob_.GlobEscape().
194	def _BackslashEscape(s):
195	# type: (str) -> str
196	"""Double up backslashes.
197
198	Useful for strings about to be globbed and strings about to be IFS
199	escaped.
200	"""
201	return s.replace('\\', '\\\\')
202
203
204	def _ValueToPartValue(val, quoted, part_loc):
205	# type: (value_t, bool, word_part_t) -> part_value_t
206	"""Helper for VarSub evaluation.
207
208	Called by _EvalBracedVarSub and _EvalWordPart for SimpleVarSub.
209	"""
210	UP_val = val
211
212	with tagswitch(val) as case:
213	if case(value_e.Undef):
214	# This happens in the case of ${undef+foo}. We skipped _EmptyStrOrError,
215	# but we have to append to the empty string.
216	return Piece('', quoted, not quoted)
217
218	elif case(value_e.Str):
219	val = cast(value.Str, UP_val)
220	return Piece(val.s, quoted, not quoted)
221
222	elif case(value_e.BashArray):
223	val = cast(value.BashArray, UP_val)
224	return part_value.Array(val.strs)
225
226	elif case(value_e.BashAssoc):
227	val = cast(value.BashAssoc, UP_val)
228	# bash behavior: splice values!
229	return part_value.Array(val.d.values())
230
231	# Cases added for YSH
232	# value_e.List is also here - we use val_ops.stringify()s err message
233	elif case(value_e.Null, value_e.Bool, value_e.Int, value_e.Float,
234	value_e.Eggex, value_e.List):
235	s = val_ops.Stringify(val, loc.Missing)
236	return Piece(s, quoted, not quoted)
237
238	else:
239	raise error.TypeErr(val, "Can't substitute into word",
240	loc.WordPart(part_loc))
241
242	raise AssertionError('for -Wreturn-type in C++')
243
244
245	def _MakeWordFrames(part_vals):
246	# type: (List[part_value_t]) -> List[List[Piece]]
247	"""A word evaluates to a flat list of part_value (String or Array). frame
248	is a portion that results in zero or more args. It can never be joined.
249	This idea exists because of arrays like "$@" and "${a[@]}".
250
251	Example:
252
253	a=(1 '2 3' 4)
254	x=x
255	y=y
256
257	# This word
258	$x"${a[@]}"$y
259
260	# Results in Three frames:
261	[ ('x', False, True), ('1', True, False) ]
262	[ ('2 3', True, False) ]
263	[ ('4', True, False), ('y', False, True) ]
264
265	Note: A frame is a 3-tuple that's identical to Piece()? Maybe we
266	should make that top level type.
267
268	TODO:
269	- Instead of List[List[Piece]], where List[Piece] is a Frame
270	- Change this representation to
271	Frames = (List[Piece] pieces, List[int] break_indices)
272	# where break_indices are the end
273
274	Consider a common case like "$x" or "${x}" - I think this a lot more
275	efficient?
276
277	And then change _EvalWordFrame(pieces: List[Piece], start: int, end: int)
278	"""
279	current = [] # type: List[Piece]
280	frames = [current]
281
282	for p in part_vals:
283	UP_p = p
284
285	with tagswitch(p) as case:
286	if case(part_value_e.String):
287	p = cast(Piece, UP_p)
288	current.append(p)
289
290	elif case(part_value_e.Array):
291	p = cast(part_value.Array, UP_p)
292
293	is_first = True
294	for s in p.strs:
295	if s is None:
296	continue # ignore undefined array entries
297
298	# Arrays parts are always quoted; otherwise they would have decayed to
299	# a string.
300	piece = Piece(s, True, False)
301	if is_first:
302	current.append(piece)
303	is_first = False
304	else:
305	current = [piece]
306	frames.append(current) # singleton frame
307
308	else:
309	raise AssertionError()
310
311	return frames
312
313
314	# TODO: This could be _MakeWordFrames and then sep.join(). It's redundant.
315	def _DecayPartValuesToString(part_vals, join_char):
316	# type: (List[part_value_t], str) -> str
317	# Decay ${a=x"$@"x} to string.
318	out = [] # type: List[str]
319	for p in part_vals:
320	UP_p = p
321	with tagswitch(p) as case:
322	if case(part_value_e.String):
323	p = cast(Piece, UP_p)
324	out.append(p.s)
325	elif case(part_value_e.Array):
326	p = cast(part_value.Array, UP_p)
327	# TODO: Eliminate double join for speed?
328	tmp = [s for s in p.strs if s is not None]
329	out.append(join_char.join(tmp))
330	else:
331	raise AssertionError()
332	return ''.join(out)
333
334
335	def _PerformSlice(
336	val, # type: value_t
337	begin, # type: int
338	length, # type: int
339	has_length, # type: bool
340	part, # type: BracedVarSub
341	arg0_val, # type: value.Str
342	):
343	# type: (...) -> value_t
344	UP_val = val
345	with tagswitch(val) as case:
346	if case(value_e.Str): # Slice UTF-8 characters in a string.
347	val = cast(value.Str, UP_val)
348	s = val.s
349	n = len(s)
350
351	if begin < 0: # Compute offset with unicode
352	byte_begin = n
353	num_iters = -begin
354	for _ in xrange(num_iters):
355	byte_begin = string_ops.PreviousUtf8Char(s, byte_begin)
356	else:
357	byte_begin = string_ops.AdvanceUtf8Chars(s, begin, 0)
358
359	if has_length:
360	if length < 0: # Compute offset with unicode
361	# Confusing: this is a POSITION
362	byte_end = n
363	num_iters = -length
364	for _ in xrange(num_iters):
365	byte_end = string_ops.PreviousUtf8Char(s, byte_end)
366	else:
367	byte_end = string_ops.AdvanceUtf8Chars(
368	s, length, byte_begin)
369	else:
370	byte_end = len(s)
371
372	substr = s[byte_begin:byte_end]
373	result = value.Str(substr) # type: value_t
374
375	elif case(value_e.BashArray): # Slice array entries.
376	val = cast(value.BashArray, UP_val)
377	# NOTE: This error is ALWAYS fatal in bash. It's inconsistent with
378	# strings.
379	if has_length and length < 0:
380	e_die(
381	"The length index of a array slice can't be negative: %d" %
382	length, loc.WordPart(part))
383
384	# Quirk: "begin" for positional arguments ($@ and $*) counts $0.
385	if arg0_val is not None:
386	orig = [arg0_val.s]
387	orig.extend(val.strs)
388	else:
389	orig = val.strs
390
391	n = len(orig)
392	if begin < 0:
393	i = n + begin # ${@:-3} starts counts from the end
394	else:
395	i = begin
396	strs = [] # type: List[str]
397	count = 0
398	while i < n:
399	if has_length and count == length: # length could be 0
400	break
401	s = orig[i]
402	if s is not None: # Unset elements don't count towards the length
403	strs.append(s)
404	count += 1
405	i += 1
406
407	result = value.BashArray(strs)
408
409	elif case(value_e.BashAssoc):
410	e_die("Can't slice associative arrays", loc.WordPart(part))
411
412	else:
413	raise error.TypeErr(val, 'Slice op expected Str or BashArray',
414	loc.WordPart(part))
415
416	return result
417
418
419	class StringWordEvaluator(object):
420	"""Interface used by ArithEvaluator / BoolEvaluator"""
421
422	def __init__(self):
423	# type: () -> None
424	"""Empty constructor for mycpp."""
425	pass
426
427	def EvalWordToString(self, w, eval_flags=0):
428	# type: (word_t, int) -> value.Str
429	raise NotImplementedError()
430
431
432	def _GetDollarHyphen(exec_opts):
433	# type: (optview.Exec) -> str
434	chars = [] # type: List[str]
435	if exec_opts.interactive():
436	chars.append('i')
437
438	if exec_opts.errexit():
439	chars.append('e')
440	if exec_opts.noglob():
441	chars.append('f')
442	if exec_opts.noexec():
443	chars.append('n')
444	if exec_opts.nounset():
445	chars.append('u')
446	# NO letter for pipefail?
447	if exec_opts.xtrace():
448	chars.append('x')
449	if exec_opts.noclobber():
450	chars.append('C')
451
452	# bash has:
453	# - c for sh -c, i for sh -i (mksh also has this)
454	# - h for hashing (mksh also has this)
455	# - B for brace expansion
456	return ''.join(chars)
457
458
459	class TildeEvaluator(object):
460
461	def __init__(self, mem, exec_opts):
462	# type: (Mem, optview.Exec) -> None
463	self.mem = mem
464	self.exec_opts = exec_opts
465
466	def GetMyHomeDir(self):
467	# type: () -> Optional[str]
468	"""Consult $HOME first, and then make a libc call.
469
470	Important: the libc call can FAIL, which is why we prefer $HOME. See issue
471	#1578.
472	"""
473	# First look up the HOME var, then ask the OS. This is what bash does.
474	val = self.mem.GetValue('HOME')
475	UP_val = val
476	if val.tag() == value_e.Str:
477	val = cast(value.Str, UP_val)
478	return val.s
479	return pyos.GetMyHomeDir()
480
481	def Eval(self, part):
482	# type: (word_part.TildeSub) -> str
483	"""Evaluates ~ and ~user, given a Lit_TildeLike token."""
484
485	if part.user_name is None:
486	result = self.GetMyHomeDir()
487	else:
488	result = pyos.GetHomeDir(part.user_name)
489
490	if result is None:
491	if self.exec_opts.strict_tilde():
492	e_die("Error expanding tilde (e.g. invalid user)", part.left)
493	else:
494	# Return ~ or ~user literally
495	result = '~'
496	if part.user_name is not None:
497	result = result + part.user_name # mycpp doesn't have +=
498
499	return result
500
501
502	class AbstractWordEvaluator(StringWordEvaluator):
503	"""Abstract base class for word evaluators.
504
505	Public entry points:
506	EvalWordToString EvalForPlugin EvalRhsWord
507	EvalWordSequence EvalWordSequence2
508	"""
509
510	def __init__(
511	self,
512	mem, # type: state.Mem
513	exec_opts, # type: optview.Exec
514	mutable_opts, # type: state.MutableOpts
515	tilde_ev, # type: TildeEvaluator
516	splitter, # type: SplitContext
517	errfmt, # type: ui.ErrorFormatter
518	):
519	# type: (...) -> None
520	self.arith_ev = None # type: sh_expr_eval.ArithEvaluator
521	self.expr_ev = None # type: expr_eval.ExprEvaluator
522	self.prompt_ev = None # type: prompt.Evaluator
523
524	self.unsafe_arith = None # type: sh_expr_eval.UnsafeArith
525
526	self.tilde_ev = tilde_ev
527
528	self.mem = mem # for $HOME, $1, etc.
529	self.exec_opts = exec_opts # for nounset
530	self.mutable_opts = mutable_opts # for _allow_command_sub
531	self.splitter = splitter
532	self.errfmt = errfmt
533
534	self.globber = glob_.Globber(exec_opts)
535
536	def CheckCircularDeps(self):
537	# type: () -> None
538	raise NotImplementedError()
539
540	def _EvalCommandSub(self, cs_part, quoted):
541	# type: (CommandSub, bool) -> part_value_t
542	"""Abstract since it has a side effect."""
543	raise NotImplementedError()
544
545	def _EvalProcessSub(self, cs_part):
546	# type: (CommandSub) -> part_value_t
547	"""Abstract since it has a side effect."""
548	raise NotImplementedError()
549
550	def _EvalVarNum(self, var_num):
551	# type: (int) -> value_t
552	assert var_num >= 0
553	return self.mem.GetArgNum(var_num)
554
555	def _EvalSpecialVar(self, op_id, quoted, vsub_state):
556	# type: (int, bool, VarSubState) -> value_t
557	"""Evaluate $?
558
559	and so forth
560	"""
561	# $@ is special -- it need to know whether it is in a double quoted
562	# context.
563	#
564	# - If it's $@ in a double quoted context, return an ARRAY.
565	# - If it's $@ in a normal context, return a STRING, which then will be
566	# subject to splitting.
567
568	if op_id in (Id.VSub_At, Id.VSub_Star):
569	argv = self.mem.GetArgv()
570	val = value.BashArray(argv) # type: value_t
571	if op_id == Id.VSub_At:
572	# "$@" evaluates to an array, $@ should be decayed
573	vsub_state.join_array = not quoted
574	else: # $* "$*" are both decayed
575	vsub_state.join_array = True
576
577	elif op_id == Id.VSub_Hyphen:
578	val = value.Str(_GetDollarHyphen(self.exec_opts))
579
580	else:
581	val = self.mem.GetSpecialVar(op_id)
582
583	return val
584
585	def _ApplyTestOp(
586	self,
587	val, # type: value_t
588	op, # type: suffix_op.Unary
589	quoted, # type: bool
590	part_vals, # type: Optional[List[part_value_t]]
591	vtest_place, # type: VTestPlace
592	blame_token, # type: Token
593	):
594	# type: (...) -> bool
595	"""
596	Returns:
597	Whether part_vals was mutated
598
599	${a:-} returns part_value[]
600	${a:+} returns part_value[]
601	${a:?error} returns error word?
602	${a:=} returns part_value[] but also needs self.mem for side effects.
603
604	So I guess it should return part_value[], and then a flag for raising an
605	error, and then a flag for assigning it?
606	The original BracedVarSub will have the name.
607
608	Example of needing multiple part_value[]
609
610	echo X-${a:-'def'"ault"}-X
611
612	We return two part values from the BracedVarSub. Also consider:
613
614	echo ${a:-x"$@"x}
615	"""
616	eval_flags = IS_SUBST
617	if quoted:
618	eval_flags \|= QUOTED
619
620	tok = op.op
621	# NOTE: Splicing part_values is necessary because of code like
622	# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
623	# do_glob/do_elide setting.
624	UP_val = val
625	with tagswitch(val) as case:
626	if case(value_e.Undef):
627	is_falsey = True
628	elif case(value_e.Str):
629	val = cast(value.Str, UP_val)
630	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_ColonEquals,
631	Id.VTest_ColonQMark, Id.VTest_ColonPlus):
632	is_falsey = len(val.s) == 0
633	else:
634	is_falsey = False
635	elif case(value_e.BashArray):
636	val = cast(value.BashArray, UP_val)
637	# TODO: allow undefined
638	is_falsey = len(val.strs) == 0
639	elif case(value_e.BashAssoc):
640	val = cast(value.BashAssoc, UP_val)
641	is_falsey = len(val.d) == 0
642	else:
643	# value.Eggex, etc. are all false
644	is_falsey = False
645
646	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_Hyphen):
647	if is_falsey:
648	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
649	return True
650	else:
651	return False
652
653	# Inverse of the above.
654	elif tok.id in (Id.VTest_ColonPlus, Id.VTest_Plus):
655	if is_falsey:
656	return False
657	else:
658	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
659	return True
660
661	# Splice and assign
662	elif tok.id in (Id.VTest_ColonEquals, Id.VTest_Equals):
663	if is_falsey:
664	# Collect new part vals.
665	assign_part_vals = [] # type: List[part_value_t]
666	self._EvalRhsWordToParts(op.arg_word, assign_part_vals,
667	eval_flags)
668	# Append them to out param AND return them.
669	part_vals.extend(assign_part_vals)
670
671	if vtest_place.name is None:
672	# TODO: error context
673	e_die("Can't assign to special variable")
674	else:
675	# NOTE: This decays arrays too! 'shopt -s strict_array' could
676	# avoid it.
677	rhs_str = _DecayPartValuesToString(
678	assign_part_vals, self.splitter.GetJoinChar())
679	if vtest_place.index is None: # using None when no index
680	lval = location.LName(
681	vtest_place.name) # type: sh_lvalue_t
682	else:
683	var_name = vtest_place.name
684	var_index = vtest_place.index
685	UP_var_index = var_index
686
687	with tagswitch(var_index) as case:
688	if case(a_index_e.Int):
689	var_index = cast(a_index.Int, UP_var_index)
690	lval = sh_lvalue.Indexed(
691	var_name, var_index.i, loc.Missing)
692	elif case(a_index_e.Str):
693	var_index = cast(a_index.Str, UP_var_index)
694	lval = sh_lvalue.Keyed(var_name, var_index.s,
695	loc.Missing)
696	else:
697	raise AssertionError()
698
699	state.OshLanguageSetValue(self.mem, lval,
700	value.Str(rhs_str))
701	return True
702
703	else:
704	return False
705
706	elif tok.id in (Id.VTest_ColonQMark, Id.VTest_QMark):
707	if is_falsey:
708	# The arg is the error message
709	error_part_vals = [] # type: List[part_value_t]
710	self._EvalRhsWordToParts(op.arg_word, error_part_vals,
711	eval_flags)
712	error_str = _DecayPartValuesToString(
713	error_part_vals, self.splitter.GetJoinChar())
714	e_die("unset variable %r" % error_str, blame_token)
715
716	else:
717	return False
718
719	else:
720	raise AssertionError(tok.id)
721
722	def _Length(self, val, token):
723	# type: (value_t, Token) -> int
724	"""Returns the length of the value, for ${#var}"""
725	UP_val = val
726	with tagswitch(val) as case:
727	if case(value_e.Str):
728	val = cast(value.Str, UP_val)
729	# NOTE: Whether bash counts bytes or chars is affected by LANG
730	# environment variables.
731	# Should we respect that, or another way to select? set -o
732	# count-bytes?
733
734	# https://stackoverflow.com/questions/17368067/length-of-string-in-bash
735	try:
736	length = string_ops.CountUtf8Chars(val.s)
737	except error.Strict as e:
738	# Add this here so we don't have to add it so far down the stack.
739	# TODO: It's better to show BOTH this CODE an the actual DATA
740	# somehow.
741	e.location = token
742
743	if self.exec_opts.strict_word_eval():
744	raise
745	else:
746	# NOTE: Doesn't make the command exit with 1; it just returns a
747	# length of -1.
748	self.errfmt.PrettyPrintError(e, prefix='warning: ')
749	return -1
750
751	elif case(value_e.BashArray):
752	val = cast(value.BashArray, UP_val)
753	# There can be empty placeholder values in the array.
754	length = 0
755	for s in val.strs:
756	if s is not None:
757	length += 1
758
759	elif case(value_e.BashAssoc):
760	val = cast(value.BashAssoc, UP_val)
761	length = len(val.d)
762
763	else:
764	raise error.TypeErr(
765	val, "Length op expected Str, BashArray, BashAssoc", token)
766
767	return length
768
769	def _Keys(self, val, token):
770	# type: (value_t, Token) -> value_t
771	"""Return keys of a container, for ${!array[@]}"""
772
773	UP_val = val
774	with tagswitch(val) as case:
775	if case(value_e.BashArray):
776	val = cast(value.BashArray, UP_val)
777	# translation issue: tuple indices not supported in list comprehensions
778	#indices = [str(i) for i, s in enumerate(val.strs) if s is not None]
779	indices = [] # type: List[str]
780	for i, s in enumerate(val.strs):
781	if s is not None:
782	indices.append(str(i))
783	return value.BashArray(indices)
784
785	elif case(value_e.BashAssoc):
786	val = cast(value.BashAssoc, UP_val)
787	assert val.d is not None # for MyPy, so it's not Optional[]
788
789	# BUG: Keys aren't ordered according to insertion!
790	return value.BashArray(val.d.keys())
791
792	else:
793	raise error.TypeErr(val, 'Keys op expected Str', token)
794
795	def _EvalVarRef(self, val, blame_tok, quoted, vsub_state, vtest_place):
796	# type: (value_t, Token, bool, VarSubState, VTestPlace) -> value_t
797	"""Handles indirect expansion like ${!var} and ${!a[0]}.
798
799	Args:
800	blame_tok: 'foo' for ${!foo}
801	"""
802	UP_val = val
803	with tagswitch(val) as case:
804	if case(value_e.Undef):
805	return value.Undef # ${!undef} is just weird bash behavior
806
807	elif case(value_e.Str):
808	val = cast(value.Str, UP_val)
809	bvs_part = self.unsafe_arith.ParseVarRef(val.s, blame_tok)
810	return self._VarRefValue(bvs_part, quoted, vsub_state,
811	vtest_place)
812
813	elif case(value_e.BashArray): # caught earlier but OK
814	e_die('Indirect expansion of array')
815
816	elif case(value_e.BashAssoc): # caught earlier but OK
817	e_die('Indirect expansion of assoc array')
818
819	else:
820	raise error.TypeErr(val, 'Var Ref op expected Str', blame_tok)
821
822	def _ApplyUnarySuffixOp(self, val, op):
823	# type: (value_t, suffix_op.Unary) -> value_t
824	assert val.tag() != value_e.Undef
825
826	op_kind = consts.GetKind(op.op.id)
827
828	if op_kind == Kind.VOp1:
829	# NOTE: glob syntax is supported in ^ ^^ , ,, ! As well as % %% # ##.
830	# Detect has_extglob so that DoUnarySuffixOp doesn't use the fast
831	# shortcut for constant strings.
832	arg_val, has_extglob = self.EvalWordToPattern(op.arg_word)
833	assert arg_val.tag() == value_e.Str
834
835	UP_val = val
836	with tagswitch(val) as case:
837	if case(value_e.Str):
838	val = cast(value.Str, UP_val)
839	s = string_ops.DoUnarySuffixOp(val.s, op.op, arg_val.s,
840	has_extglob)
841	#log('%r %r -> %r', val.s, arg_val.s, s)
842	new_val = value.Str(s) # type: value_t
843
844	elif case(value_e.BashArray):
845	val = cast(value.BashArray, UP_val)
846	# ${a[@]#prefix} is VECTORIZED on arrays. YSH should have this too.
847	strs = [] # type: List[str]
848	for s in val.strs:
849	if s is not None:
850	strs.append(
851	string_ops.DoUnarySuffixOp(
852	s, op.op, arg_val.s, has_extglob))
853	new_val = value.BashArray(strs)
854
855	elif case(value_e.BashAssoc):
856	val = cast(value.BashAssoc, UP_val)
857	strs = []
858	for s in val.d.values():
859	strs.append(
860	string_ops.DoUnarySuffixOp(s, op.op, arg_val.s,
861	has_extglob))
862	new_val = value.BashArray(strs)
863
864	else:
865	raise error.TypeErr(
866	val, 'Unary op expected Str, BashArray, BashAssoc',
867	op.op)
868
869	else:
870	raise AssertionError(Kind_str(op_kind))
871
872	return new_val
873
874	def _PatSub(self, val, op):
875	# type: (value_t, suffix_op.PatSub) -> value_t
876
877	pat_val, has_extglob = self.EvalWordToPattern(op.pat)
878	# Extended globs aren't supported because we only translate * ? etc. to
879	# ERE. I don't think there's a straightforward translation from !(*.py) to
880	# ERE! You would need an engine that supports negation? (Derivatives?)
881	if has_extglob:
882	e_die('extended globs not supported in ${x//GLOB/}', op.pat)
883
884	if op.replace:
885	replace_val = self.EvalRhsWord(op.replace)
886	# Can't have an array, so must be a string
887	assert replace_val.tag() == value_e.Str, replace_val
888	replace_str = cast(value.Str, replace_val).s
889	else:
890	replace_str = ''
891
892	# note: doesn't support self.exec_opts.extglob()!
893	regex, warnings = glob_.GlobToERE(pat_val.s)
894	if len(warnings):
895	# TODO:
896	# - Add 'shopt -s strict_glob' mode and expose warnings.
897	# "Glob is not in CANONICAL FORM".
898	# - Propagate location info back to the 'op.pat' word.
899	pass
900	#log('regex %r', regex)
901	replacer = string_ops.GlobReplacer(regex, replace_str, op.slash_tok)
902
903	with tagswitch(val) as case2:
904	if case2(value_e.Str):
905	str_val = cast(value.Str, val)
906	s = replacer.Replace(str_val.s, op)
907	val = value.Str(s)
908
909	elif case2(value_e.BashArray):
910	array_val = cast(value.BashArray, val)
911	strs = [] # type: List[str]
912	for s in array_val.strs:
913	if s is not None:
914	strs.append(replacer.Replace(s, op))
915	val = value.BashArray(strs)
916
917	elif case2(value_e.BashAssoc):
918	assoc_val = cast(value.BashAssoc, val)
919	strs = []
920	for s in assoc_val.d.values():
921	strs.append(replacer.Replace(s, op))
922	val = value.BashArray(strs)
923
924	else:
925	raise error.TypeErr(
926	val, 'Pat Sub op expected Str, BashArray, BashAssoc',
927	op.slash_tok)
928
929	return val
930
931	def _Slice(self, val, op, var_name, part):
932	# type: (value_t, suffix_op.Slice, Optional[str], BracedVarSub) -> value_t
933
934	if op.begin:
935	begin = self.arith_ev.EvalToInt(op.begin)
936	else:
937	begin = 0
938
939	# Note: bash allows lengths to be negative (with odd semantics), but
940	# we don't allow that right now.
941	has_length = False
942	length = -1
943	if op.length:
944	has_length = True
945	length = self.arith_ev.EvalToInt(op.length)
946
947	try:
948	arg0_val = None # type: value.Str
949	if var_name is None: # $* or $@
950	arg0_val = self.mem.GetArg0()
951	val = _PerformSlice(val, begin, length, has_length, part, arg0_val)
952	except error.Strict as e:
953	if self.exec_opts.strict_word_eval():
954	raise
955	else:
956	self.errfmt.PrettyPrintError(e, prefix='warning: ')
957	with tagswitch(val) as case2:
958	if case2(value_e.Str):
959	val = value.Str('')
960	elif case2(value_e.BashArray):
961	val = value.BashArray([])
962	else:
963	raise NotImplementedError()
964	return val
965
966	def _Nullary(self, val, op, var_name):
967	# type: (value_t, Token, Optional[str]) -> Tuple[value.Str, bool]
968
969	UP_val = val
970	quoted2 = False
971	op_id = op.id
972	if op_id == Id.VOp0_P:
973	with tagswitch(val) as case:
974	if case(value_e.Str):
975	str_val = cast(value.Str, UP_val)
976	prompt = self.prompt_ev.EvalPrompt(str_val)
977	# readline gets rid of these, so we should too.
978	p = prompt.replace('\x01', '').replace('\x02', '')
979	result = value.Str(p)
980	else:
981	e_die("Can't use @P on %s" % ui.ValType(val), op)
982
983	elif op_id == Id.VOp0_Q:
984	with tagswitch(val) as case:
985	if case(value_e.Str):
986	str_val = cast(value.Str, UP_val)
987
988	# TODO: use fastfunc.ShellEncode or
989	# fastfunc.PosixShellEncode()
990	result = value.Str(j8_lite.MaybeShellEncode(str_val.s))
991	# oddly, 'echo ${x@Q}' is equivalent to 'echo "${x@Q}"' in bash
992	quoted2 = True
993	elif case(value_e.BashArray):
994	array_val = cast(value.BashArray, UP_val)
995
996	# TODO: should use fastfunc.ShellEncode
997	tmp = [j8_lite.MaybeShellEncode(s) for s in array_val.strs]
998	result = value.Str(' '.join(tmp))
999	else:
1000	e_die("Can't use @Q on %s" % ui.ValType(val), op)
1001
1002	elif op_id == Id.VOp0_a:
1003	# We're ONLY simluating -a and -A, not -r -x -n for now. See
1004	# spec/ble-idioms.test.sh.
1005	chars = [] # type: List[str]
1006	with tagswitch(val) as case:
1007	if case(value_e.BashArray):
1008	chars.append('a')
1009	elif case(value_e.BashAssoc):
1010	chars.append('A')
1011
1012	if var_name is not None: # e.g. ${?@a} is allowed
1013	cell = self.mem.GetCell(var_name)
1014	if cell:
1015	if cell.readonly:
1016	chars.append('r')
1017	if cell.exported:
1018	chars.append('x')
1019	if cell.nameref:
1020	chars.append('n')
1021
1022	result = value.Str(''.join(chars))
1023
1024	else:
1025	e_die('Var op %r not implemented' % lexer.TokenVal(op), op)
1026
1027	return result, quoted2
1028
1029	def _WholeArray(self, val, part, quoted, vsub_state):
1030	# type: (value_t, BracedVarSub, bool, VarSubState) -> value_t
1031	op_id = cast(bracket_op.WholeArray, part.bracket_op).op_id
1032
1033	if op_id == Id.Lit_At:
1034	vsub_state.join_array = not quoted # ${a[@]} decays but "${a[@]}" doesn't
1035	UP_val = val
1036	with tagswitch(val) as case2:
1037	if case2(value_e.Undef):
1038	if not vsub_state.has_test_op:
1039	val = self._EmptyBashArrayOrError(part.token)
1040	elif case2(value_e.Str):
1041	if self.exec_opts.strict_array():
1042	e_die("Can't index string with @", loc.WordPart(part))
1043	elif case2(value_e.BashArray):
1044	pass # no-op
1045
1046	elif op_id == Id.Arith_Star:
1047	vsub_state.join_array = True # both ${a[]} and "${a[]}" decay
1048	UP_val = val
1049	with tagswitch(val) as case2:
1050	if case2(value_e.Undef):
1051	if not vsub_state.has_test_op:
1052	val = self._EmptyBashArrayOrError(part.token)
1053	elif case2(value_e.Str):
1054	if self.exec_opts.strict_array():
1055	e_die("Can't index string with *", loc.WordPart(part))
1056	elif case2(value_e.BashArray):
1057	pass # no-op
1058
1059	else:
1060	raise AssertionError(op_id) # unknown
1061
1062	return val
1063
1064	def _ArrayIndex(self, val, part, vtest_place):
1065	# type: (value_t, BracedVarSub, VTestPlace) -> value_t
1066	"""Process a numeric array index like ${a[i+1]}"""
1067	anode = cast(bracket_op.ArrayIndex, part.bracket_op).expr
1068
1069	UP_val = val
1070	with tagswitch(val) as case2:
1071	if case2(value_e.Undef):
1072	pass # it will be checked later
1073
1074	elif case2(value_e.Str):
1075	# Bash treats any string as an array, so we can't add our own
1076	# behavior here without making valid OSH invalid bash.
1077	e_die("Can't index string %r with integer" % part.var_name,
1078	part.token)
1079
1080	elif case2(value_e.BashArray):
1081	array_val = cast(value.BashArray, UP_val)
1082	index = self.arith_ev.EvalToInt(anode)
1083	vtest_place.index = a_index.Int(index)
1084
1085	s = GetArrayItem(array_val.strs, index)
1086
1087	if s is None:
1088	val = value.Undef
1089	else:
1090	val = value.Str(s)
1091
1092	elif case2(value_e.BashAssoc):
1093	assoc_val = cast(value.BashAssoc, UP_val)
1094	# Location could also be attached to bracket_op? But
1095	# arith_expr.VarSub works OK too
1096	key = self.arith_ev.EvalWordToString(
1097	anode, blame_loc=location.TokenForArith(anode))
1098
1099	vtest_place.index = a_index.Str(key) # out param
1100	s = assoc_val.d.get(key)
1101
1102	if s is None:
1103	val = value.Undef
1104	else:
1105	val = value.Str(s)
1106
1107	else:
1108	raise error.TypeErr(val,
1109	'Index op expected BashArray, BashAssoc',
1110	loc.WordPart(part))
1111
1112	return val
1113
1114	def _EvalDoubleQuoted(self, parts, part_vals):
1115	# type: (List[word_part_t], List[part_value_t]) -> None
1116	"""Evaluate parts of a DoubleQuoted part.
1117
1118	Args:
1119	part_vals: output param to append to.
1120	"""
1121	# Example of returning array:
1122	# $ a=(1 2); b=(3); $ c=(4 5)
1123	# $ argv "${a[@]}${b[@]}${c[@]}"
1124	# ['1', '234', '5']
1125	#
1126	# Example of multiple parts
1127	# $ argv "${a[@]}${undef[@]:-${c[@]}}"
1128	# ['1', '24', '5']
1129
1130	# Special case for "". The parser outputs (DoubleQuoted []), instead
1131	# of (DoubleQuoted [Literal '']). This is better but it means we
1132	# have to check for it.
1133	if len(parts) == 0:
1134	v = Piece('', True, False)
1135	part_vals.append(v)
1136	return
1137
1138	for p in parts:
1139	self._EvalWordPart(p, part_vals, QUOTED)
1140
1141	def EvalDoubleQuotedToString(self, dq_part):
1142	# type: (DoubleQuoted) -> str
1143	"""For double quoted strings in YSH expressions.
1144
1145	Example: var x = "$foo-${foo}"
1146	"""
1147	part_vals = [] # type: List[part_value_t]
1148	self._EvalDoubleQuoted(dq_part.parts, part_vals)
1149	return self._ConcatPartVals(part_vals, dq_part.left)
1150
1151	def _DecayArray(self, val):
1152	# type: (value.BashArray) -> value.Str
1153	"""Decay $* to a string."""
1154	assert val.tag() == value_e.BashArray, val
1155	sep = self.splitter.GetJoinChar()
1156	tmp = [s for s in val.strs if s is not None]
1157	return value.Str(sep.join(tmp))
1158
1159	def _EmptyStrOrError(self, val, token):
1160	# type: (value_t, Token) -> value_t
1161	if val.tag() != value_e.Undef:
1162	return val
1163
1164	if not self.exec_opts.nounset():
1165	return value.Str('')
1166
1167	tok_str = lexer.TokenVal(token)
1168	name = tok_str[1:] if tok_str.startswith('$') else tok_str
1169	e_die('Undefined variable %r' % name, token)
1170
1171	def _EmptyBashArrayOrError(self, token):
1172	# type: (Token) -> value_t
1173	assert token is not None
1174	if self.exec_opts.nounset():
1175	e_die('Undefined array %r' % lexer.TokenVal(token), token)
1176	else:
1177	return value.BashArray([])
1178
1179	def _EvalBracketOp(self, val, part, quoted, vsub_state, vtest_place):
1180	# type: (value_t, BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1181
1182	if part.bracket_op:
1183	with tagswitch(part.bracket_op) as case:
1184	if case(bracket_op_e.WholeArray):
1185	val = self._WholeArray(val, part, quoted, vsub_state)
1186
1187	elif case(bracket_op_e.ArrayIndex):
1188	val = self._ArrayIndex(val, part, vtest_place)
1189
1190	else:
1191	raise AssertionError(part.bracket_op.tag())
1192
1193	else: # no bracket op
1194	var_name = vtest_place.name
1195	if (var_name is not None and
1196	val.tag() in (value_e.BashArray, value_e.BashAssoc) and
1197	not vsub_state.is_type_query):
1198	if ShouldArrayDecay(var_name, self.exec_opts,
1199	not (part.prefix_op or part.suffix_op)):
1200	# for ${BASH_SOURCE}, etc.
1201	val = DecayArray(val)
1202	else:
1203	e_die(
1204	"Array %r can't be referred to as a scalar (without @ or *)"
1205	% var_name, loc.WordPart(part))
1206
1207	return val
1208
1209	def _VarRefValue(self, part, quoted, vsub_state, vtest_place):
1210	# type: (BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1211	"""Duplicates some logic from _EvalBracedVarSub, but returns a
1212	value_t."""
1213
1214	# 1. Evaluate from (var_name, var_num, token Id) -> value
1215	if part.token.id == Id.VSub_Name:
1216	vtest_place.name = part.var_name
1217	val = self.mem.GetValue(part.var_name)
1218
1219	elif part.token.id == Id.VSub_Number:
1220	var_num = int(part.var_name)
1221	val = self._EvalVarNum(var_num)
1222
1223	else:
1224	# $* decays
1225	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1226
1227	# We don't need var_index because it's only for L-Values of test ops?
1228	if self.exec_opts.eval_unsafe_arith():
1229	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1230	vtest_place)
1231	else:
1232	with state.ctx_Option(self.mutable_opts,
1233	[option_i._allow_command_sub], False):
1234	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1235	vtest_place)
1236
1237	return val
1238
1239	def _EvalBracedVarSub(self, part, part_vals, quoted):
1240	# type: (BracedVarSub, List[part_value_t], bool) -> None
1241	"""
1242	Args:
1243	part_vals: output param to append to.
1244	"""
1245	# We have different operators that interact in a non-obvious order.
1246	#
1247	# 1. bracket_op: value -> value, with side effect on vsub_state
1248	#
1249	# 2. prefix_op
1250	# a. length ${#x}: value -> value
1251	# b. var ref ${!ref}: can expand to an array
1252	#
1253	# 3. suffix_op:
1254	# a. no operator: you have a value
1255	# b. Test: value -> part_value[]
1256	# c. Other Suffix: value -> value
1257	#
1258	# 4. Process vsub_state.join_array here before returning.
1259	#
1260	# These cases are hard to distinguish:
1261	# - ${!prefix@} prefix query
1262	# - ${!array[@]} keys
1263	# - ${!ref} named reference
1264	# - ${!ref[0]} named reference
1265	#
1266	# I think we need several stages:
1267	#
1268	# 1. value: name, number, special, prefix query
1269	# 2. bracket_op
1270	# 3. prefix length -- this is TERMINAL
1271	# 4. indirection? Only for some of the ! cases
1272	# 5. string transformation suffix ops like ##
1273	# 6. test op
1274	# 7. vsub_state.join_array
1275
1276	# vsub_state.join_array is for joining "${a[*]}" and unquoted ${a[@]} AFTER
1277	# suffix ops are applied. If we take the length with a prefix op, the
1278	# distinction is ignored.
1279
1280	var_name = None # type: Optional[str] # used throughout the function
1281	vtest_place = VTestPlace(var_name, None) # For ${foo=default}
1282	vsub_state = VarSubState.CreateNull() # for $, ${a[]}, etc.
1283
1284	# 1. Evaluate from (var_name, var_num, token Id) -> value
1285	if part.token.id == Id.VSub_Name:
1286	# Handle ${!prefix@} first, since that looks at names and not values
1287	# Do NOT handle ${!A[@]@a} here!
1288	if (part.prefix_op is not None and part.bracket_op is None and
1289	part.suffix_op is not None and
1290	part.suffix_op.tag() == suffix_op_e.Nullary):
1291	nullary_op = cast(Token, part.suffix_op)
1292	# ${!x@} but not ${!x@P}
1293	if consts.GetKind(nullary_op.id) == Kind.VOp3:
1294	names = self.mem.VarNamesStartingWith(part.var_name)
1295	names.sort()
1296
1297	if quoted and nullary_op.id == Id.VOp3_At:
1298	part_vals.append(part_value.Array(names))
1299	else:
1300	sep = self.splitter.GetJoinChar()
1301	part_vals.append(Piece(sep.join(names), quoted, True))
1302	return # EARLY RETURN
1303
1304	var_name = part.var_name
1305	vtest_place.name = var_name # for _ApplyTestOp
1306
1307	val = self.mem.GetValue(var_name)
1308
1309	elif part.token.id == Id.VSub_Number:
1310	var_num = int(part.var_name)
1311	val = self._EvalVarNum(var_num)
1312	else:
1313	# $* decays
1314	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1315
1316	suffix_op_ = part.suffix_op
1317	if suffix_op_:
1318	UP_op = suffix_op_
1319	with tagswitch(suffix_op_) as case:
1320	if case(suffix_op_e.Nullary):
1321	suffix_op_ = cast(Token, UP_op)
1322
1323	# Type query ${array@a} is a STRING, not an array
1324	# NOTE: ${array@Q} is ${array[0]@Q} in bash, which is different than
1325	# ${array[@]@Q}
1326	if suffix_op_.id == Id.VOp0_a:
1327	vsub_state.is_type_query = True
1328
1329	elif case(suffix_op_e.Unary):
1330	suffix_op_ = cast(suffix_op.Unary, UP_op)
1331
1332	# Do the _EmptyStrOrError/_EmptyBashArrayOrError up front, EXCEPT in
1333	# the case of Kind.VTest
1334	if consts.GetKind(suffix_op_.op.id) == Kind.VTest:
1335	vsub_state.has_test_op = True
1336
1337	# 2. Bracket Op
1338	val = self._EvalBracketOp(val, part, quoted, vsub_state, vtest_place)
1339
1340	if part.prefix_op:
1341	if part.prefix_op.id == Id.VSub_Pound: # ${#var} for length
1342	if not vsub_state.has_test_op: # undef -> '' BEFORE length
1343	val = self._EmptyStrOrError(val, part.token)
1344
1345	n = self._Length(val, part.token)
1346	part_vals.append(Piece(str(n), quoted, False))
1347	return # EARLY EXIT: nothing else can come after length
1348
1349	elif part.prefix_op.id == Id.VSub_Bang:
1350	if (part.bracket_op and
1351	part.bracket_op.tag() == bracket_op_e.WholeArray):
1352	if vsub_state.has_test_op:
1353	# ${!a[@]-'default'} is a non-fatal runtime error in bash. Here
1354	# it's fatal.
1355	op_tok = cast(suffix_op.Unary, UP_op).op
1356	e_die('Test operation not allowed with ${!array[@]}',
1357	op_tok)
1358
1359	# ${!array[@]} to get indices/keys
1360	val = self._Keys(val, part.token)
1361	# already set vsub_State.join_array ABOVE
1362	else:
1363	# Process ${!ref}. SURPRISE: ${!a[0]} is an indirect expansion unlike
1364	# ${!a[@]} !
1365	# ${!ref} can expand into an array if ref='array[@]'
1366
1367	# Clear it now that we have a var ref
1368	vtest_place.name = None
1369	vtest_place.index = None
1370
1371	val = self._EvalVarRef(val, part.token, quoted, vsub_state,
1372	vtest_place)
1373
1374	if not vsub_state.has_test_op: # undef -> '' AFTER indirection
1375	val = self._EmptyStrOrError(val, part.token)
1376
1377	else:
1378	raise AssertionError(part.prefix_op)
1379
1380	else:
1381	if not vsub_state.has_test_op: # undef -> '' if no prefix op
1382	val = self._EmptyStrOrError(val, part.token)
1383
1384	quoted2 = False # another bit for @Q
1385	if suffix_op_:
1386	op = suffix_op_ # could get rid of this alias
1387
1388	with tagswitch(suffix_op_) as case:
1389	if case(suffix_op_e.Nullary):
1390	op = cast(Token, UP_op)
1391	val, quoted2 = self._Nullary(val, op, var_name)
1392
1393	elif case(suffix_op_e.Unary):
1394	op = cast(suffix_op.Unary, UP_op)
1395	if consts.GetKind(op.op.id) == Kind.VTest:
1396	if self._ApplyTestOp(val, op, quoted, part_vals,
1397	vtest_place, part.token):
1398	# e.g. to evaluate ${undef:-'default'}, we already appended
1399	# what we need
1400	return
1401
1402	else:
1403	# Other suffix: value -> value
1404	val = self._ApplyUnarySuffixOp(val, op)
1405
1406	elif case(suffix_op_e.PatSub): # PatSub, vectorized
1407	op = cast(suffix_op.PatSub, UP_op)
1408	val = self._PatSub(val, op)
1409
1410	elif case(suffix_op_e.Slice):
1411	op = cast(suffix_op.Slice, UP_op)
1412	val = self._Slice(val, op, var_name, part)
1413
1414	elif case(suffix_op_e.Static):
1415	op = cast(suffix_op.Static, UP_op)
1416	e_die('Not implemented', op.tok)
1417
1418	else:
1419	raise AssertionError()
1420
1421	# After applying suffixes, process join_array here.
1422	UP_val = val
1423	if val.tag() == value_e.BashArray:
1424	array_val = cast(value.BashArray, UP_val)
1425	if vsub_state.join_array:
1426	val = self._DecayArray(array_val)
1427	else:
1428	val = array_val
1429
1430	# For example, ${a} evaluates to value.Str(), but we want a
1431	# Piece().
1432	part_val = _ValueToPartValue(val, quoted or quoted2, part)
1433	part_vals.append(part_val)
1434
1435	def _ConcatPartVals(self, part_vals, location):
1436	# type: (List[part_value_t], loc_t) -> str
1437
1438	strs = [] # type: List[str]
1439	for part_val in part_vals:
1440	UP_part_val = part_val
1441	with tagswitch(part_val) as case:
1442	if case(part_value_e.String):
1443	part_val = cast(Piece, UP_part_val)
1444	s = part_val.s
1445
1446	elif case(part_value_e.Array):
1447	part_val = cast(part_value.Array, UP_part_val)
1448	if self.exec_opts.strict_array():
1449	# Examples: echo f > "$@"; local foo="$@"
1450	e_die("Illegal array word part (strict_array)",
1451	location)
1452	else:
1453	# It appears to not respect IFS
1454	# TODO: eliminate double join()?
1455	tmp = [s for s in part_val.strs if s is not None]
1456	s = ' '.join(tmp)
1457
1458	else:
1459	raise AssertionError()
1460
1461	strs.append(s)
1462
1463	return ''.join(strs)
1464
1465	def EvalBracedVarSubToString(self, part):
1466	# type: (BracedVarSub) -> str
1467	"""For double quoted strings in YSH expressions.
1468
1469	Example: var x = "$foo-${foo}"
1470	"""
1471	part_vals = [] # type: List[part_value_t]
1472	self._EvalBracedVarSub(part, part_vals, False)
1473	# blame ${ location
1474	return self._ConcatPartVals(part_vals, part.left)
1475
1476	def _EvalSimpleVarSub(self, part, part_vals, quoted):
1477	# type: (SimpleVarSub, List[part_value_t], bool) -> None
1478
1479	token = part.tok
1480
1481	vsub_state = VarSubState.CreateNull()
1482
1483	# 1. Evaluate from (var_name, var_num, Token) -> defined, value
1484	if token.id == Id.VSub_DollarName:
1485	var_name = lexer.LazyStr(token)
1486	# TODO: Special case for LINENO
1487	val = self.mem.GetValue(var_name)
1488	if val.tag() in (value_e.BashArray, value_e.BashAssoc):
1489	if ShouldArrayDecay(var_name, self.exec_opts):
1490	# for $BASH_SOURCE, etc.
1491	val = DecayArray(val)
1492	else:
1493	e_die(
1494	"Array %r can't be referred to as a scalar (without @ or *)"
1495	% var_name, token)
1496
1497	elif token.id == Id.VSub_Number:
1498	var_num = int(lexer.LazyStr(token))
1499	val = self._EvalVarNum(var_num)
1500
1501	else:
1502	val = self._EvalSpecialVar(token.id, quoted, vsub_state)
1503
1504	#log('SIMPLE %s', part)
1505	val = self._EmptyStrOrError(val, token)
1506	UP_val = val
1507	if val.tag() == value_e.BashArray:
1508	array_val = cast(value.BashArray, UP_val)
1509	if vsub_state.join_array:
1510	val = self._DecayArray(array_val)
1511	else:
1512	val = array_val
1513
1514	v = _ValueToPartValue(val, quoted, part)
1515	part_vals.append(v)
1516
1517	def EvalSimpleVarSubToString(self, node):
1518	# type: (SimpleVarSub) -> str
1519	"""For double quoted strings in YSH expressions.
1520
1521	Example: var x = "$foo-${foo}"
1522	"""
1523	part_vals = [] # type: List[part_value_t]
1524	self._EvalSimpleVarSub(node, part_vals, False)
1525	return self._ConcatPartVals(part_vals, node.tok)
1526
1527	def _EvalExtGlob(self, part, part_vals):
1528	# type: (word_part.ExtGlob, List[part_value_t]) -> None
1529	"""Evaluate @($x\|'foo'\|$(hostname)) and flatten it."""
1530	op = part.op
1531	if op.id == Id.ExtGlob_Comma:
1532	op_str = '@('
1533	else:
1534	op_str = lexer.LazyStr(op)
1535	# Do NOT split these.
1536	part_vals.append(Piece(op_str, False, False))
1537
1538	for i, w in enumerate(part.arms):
1539	if i != 0:
1540	part_vals.append(Piece('\|', False, False)) # separator
1541	# FLATTEN the tree of extglob "arms".
1542	self._EvalWordToParts(w, part_vals, EXTGLOB_NESTED)
1543	part_vals.append(Piece(')', False, False)) # closing )
1544
1545	def _TranslateExtGlob(self, part_vals, w, glob_parts, fnmatch_parts):
1546	# type: (List[part_value_t], CompoundWord, List[str], List[str]) -> None
1547	"""Translate a flattened WORD with an ExtGlob part to string patterns.
1548
1549	We need both glob and fnmatch patterns. _EvalExtGlob does the
1550	flattening.
1551	"""
1552	for i, part_val in enumerate(part_vals):
1553	UP_part_val = part_val
1554	with tagswitch(part_val) as case:
1555	if case(part_value_e.String):
1556	part_val = cast(Piece, UP_part_val)
1557	if part_val.quoted and not self.exec_opts.noglob():
1558	s = glob_.GlobEscape(part_val.s)
1559	else:
1560	# e.g. the @( and \| in @(foo\|bar) aren't quoted
1561	s = part_val.s
1562	glob_parts.append(s)
1563	fnmatch_parts.append(s) # from _EvalExtGlob()
1564
1565	elif case(part_value_e.Array):
1566	# Disallow array
1567	e_die(
1568	"Extended globs and arrays can't appear in the same word",
1569	w)
1570
1571	elif case(part_value_e.ExtGlob):
1572	part_val = cast(part_value.ExtGlob, UP_part_val)
1573	# keep appending fnmatch_parts, but repplace glob_parts with '*'
1574	self._TranslateExtGlob(part_val.part_vals, w, [],
1575	fnmatch_parts)
1576	glob_parts.append('*')
1577
1578	else:
1579	raise AssertionError()
1580
1581	def _EvalWordPart(self, part, part_vals, flags):
1582	# type: (word_part_t, List[part_value_t], int) -> None
1583	"""Evaluate a word part, appending to part_vals
1584
1585	Called by _EvalWordToParts, EvalWordToString, and _EvalDoubleQuoted.
1586	"""
1587	quoted = bool(flags & QUOTED)
1588	is_subst = bool(flags & IS_SUBST)
1589
1590	UP_part = part
1591	with tagswitch(part) as case:
1592	if case(word_part_e.ShArrayLiteral):
1593	part = cast(ShArrayLiteral, UP_part)
1594	e_die("Unexpected array literal", loc.WordPart(part))
1595	elif case(word_part_e.BashAssocLiteral):
1596	part = cast(word_part.BashAssocLiteral, UP_part)
1597	e_die("Unexpected associative array literal",
1598	loc.WordPart(part))
1599
1600	elif case(word_part_e.Literal):
1601	part = cast(Token, UP_part)
1602	# Split if it's in a substitution.
1603	# That is: echo is not split, but ${foo:-echo} is split
1604	v = Piece(lexer.LazyStr(part), quoted, is_subst)
1605	part_vals.append(v)
1606
1607	elif case(word_part_e.EscapedLiteral):
1608	part = cast(word_part.EscapedLiteral, UP_part)
1609	v = Piece(part.ch, True, False)
1610	part_vals.append(v)
1611
1612	elif case(word_part_e.SingleQuoted):
1613	part = cast(SingleQuoted, UP_part)
1614	v = Piece(part.sval, True, False)
1615	part_vals.append(v)
1616
1617	elif case(word_part_e.DoubleQuoted):
1618	part = cast(DoubleQuoted, UP_part)
1619	self._EvalDoubleQuoted(part.parts, part_vals)
1620
1621	elif case(word_part_e.CommandSub):
1622	part = cast(CommandSub, UP_part)
1623	id_ = part.left_token.id
1624	if id_ in (Id.Left_DollarParen, Id.Left_AtParen,
1625	Id.Left_Backtick):
1626	sv = self._EvalCommandSub(part,
1627	quoted) # type: part_value_t
1628
1629	elif id_ in (Id.Left_ProcSubIn, Id.Left_ProcSubOut):
1630	sv = self._EvalProcessSub(part)
1631
1632	else:
1633	raise AssertionError(id_)
1634
1635	part_vals.append(sv)
1636
1637	elif case(word_part_e.SimpleVarSub):
1638	part = cast(SimpleVarSub, UP_part)
1639	self._EvalSimpleVarSub(part, part_vals, quoted)
1640
1641	elif case(word_part_e.BracedVarSub):
1642	part = cast(BracedVarSub, UP_part)
1643	self._EvalBracedVarSub(part, part_vals, quoted)
1644
1645	elif case(word_part_e.TildeSub):
1646	part = cast(word_part.TildeSub, UP_part)
1647	# We never parse a quoted string into a TildeSub.
1648	assert not quoted
1649	s = self.tilde_ev.Eval(part)
1650	v = Piece(s, True, False) # NOT split even when unquoted!
1651	part_vals.append(v)
1652
1653	elif case(word_part_e.ArithSub):
1654	part = cast(word_part.ArithSub, UP_part)
1655	num = self.arith_ev.EvalToBigInt(part.anode)
1656	v = Piece(mops.ToStr(num), quoted, not quoted)
1657	part_vals.append(v)
1658
1659	elif case(word_part_e.ExtGlob):
1660	part = cast(word_part.ExtGlob, UP_part)
1661	#if not self.exec_opts.extglob():
1662	# die() # disallow at runtime? Don't just decay
1663
1664	# Create a node to hold the flattened tree. The caller decides whether
1665	# to pass it to fnmatch() or replace it with '*' and pass it to glob().
1666	part_vals2 = [] # type: List[part_value_t]
1667	self._EvalExtGlob(part, part_vals2) # flattens tree
1668	part_vals.append(part_value.ExtGlob(part_vals2))
1669
1670	elif case(word_part_e.BashRegexGroup):
1671	part = cast(word_part.BashRegexGroup, UP_part)
1672
1673	part_vals.append(Piece('(', False, False)) # not quoted
1674	if part.child:
1675	self._EvalWordToParts(part.child, part_vals, 0)
1676	part_vals.append(Piece(')', False, False))
1677
1678	elif case(word_part_e.Splice):
1679	part = cast(word_part.Splice, UP_part)
1680	val = self.mem.GetValue(part.var_name)
1681
1682	strs = self.expr_ev.SpliceValue(val, part)
1683	part_vals.append(part_value.Array(strs))
1684
1685	elif case(word_part_e.ExprSub):
1686	part = cast(word_part.ExprSub, UP_part)
1687	part_val = self.expr_ev.EvalExprSub(part)
1688	part_vals.append(part_val)
1689
1690	elif case(word_part_e.ZshVarSub):
1691	part = cast(word_part.ZshVarSub, UP_part)
1692	e_die("ZSH var subs are parsed, but can't be evaluated",
1693	part.left)
1694
1695	else:
1696	raise AssertionError(part.tag())
1697
1698	def _EvalRhsWordToParts(self, w, part_vals, eval_flags=0):
1699	# type: (rhs_word_t, List[part_value_t], int) -> None
1700	quoted = bool(eval_flags & QUOTED)
1701
1702	UP_w = w
1703	with tagswitch(w) as case:
1704	if case(rhs_word_e.Empty):
1705	part_vals.append(Piece('', quoted, not quoted))
1706
1707	elif case(rhs_word_e.Compound):
1708	w = cast(CompoundWord, UP_w)
1709	self._EvalWordToParts(w, part_vals, eval_flags=eval_flags)
1710
1711	else:
1712	raise AssertionError()
1713
1714	def _EvalWordToParts(self, w, part_vals, eval_flags=0):
1715	# type: (CompoundWord, List[part_value_t], int) -> None
1716	"""Helper for EvalRhsWord, EvalWordSequence, etc.
1717
1718	Returns:
1719	Appends to part_vals. Note that this is a TREE.
1720	"""
1721	# Does the word have an extended glob? This is a special case because
1722	# of the way we use glob() and then fnmatch(..., FNM_EXTMATCH) to
1723	# implement extended globs. It's hard to carry that extra information
1724	# all the way past the word splitting stage.
1725
1726	# OSH semantic limitations: If a word has an extended glob part, then
1727	# 1. It can't have an array
1728	# 2. Word splitting of unquoted words isn't respected
1729
1730	word_part_vals = [] # type: List[part_value_t]
1731	has_extglob = False
1732	for p in w.parts:
1733	if p.tag() == word_part_e.ExtGlob:
1734	has_extglob = True
1735	self._EvalWordPart(p, word_part_vals, eval_flags)
1736
1737	# Caller REQUESTED extglob evaluation, AND we parsed word_part.ExtGlob()
1738	if has_extglob:
1739	if bool(eval_flags & EXTGLOB_FILES):
1740	# Treat the WHOLE word as a pattern. We need to TWO VARIANTS of the
1741	# word because of the way we use libc:
1742	# 1. With '*' for extglob parts
1743	# 2. With _EvalExtGlob() for extglob parts
1744
1745	glob_parts = [] # type: List[str]
1746	fnmatch_parts = [] # type: List[str]
1747	self._TranslateExtGlob(word_part_vals, w, glob_parts,
1748	fnmatch_parts)
1749
1750	#log('word_part_vals %s', word_part_vals)
1751	glob_pat = ''.join(glob_parts)
1752	fnmatch_pat = ''.join(fnmatch_parts)
1753	#log("glob %s fnmatch %s", glob_pat, fnmatch_pat)
1754
1755	results = [] # type: List[str]
1756	n = self.globber.ExpandExtended(glob_pat, fnmatch_pat, results)
1757	if n < 0:
1758	raise error.FailGlob(
1759	'Extended glob %r matched no files' % fnmatch_pat, w)
1760
1761	part_vals.append(part_value.Array(results))
1762	elif bool(eval_flags & EXTGLOB_NESTED):
1763	# We only glob at the TOP level of @(nested\|@(pattern))
1764	part_vals.extend(word_part_vals)
1765	else:
1766	# e.g. simple_word_eval, assignment builtin
1767	e_die('Extended glob not allowed in this word', w)
1768	else:
1769	part_vals.extend(word_part_vals)
1770
1771	def _PartValsToString(self, part_vals, w, eval_flags, strs):
1772	# type: (List[part_value_t], CompoundWord, int, List[str]) -> None
1773	"""Helper for EvalWordToString, similar to _ConcatPartVals() above.
1774
1775	Note: arg 'w' could just be a span ID
1776	"""
1777	for part_val in part_vals:
1778	UP_part_val = part_val
1779	with tagswitch(part_val) as case:
1780	if case(part_value_e.String):
1781	part_val = cast(Piece, UP_part_val)
1782	s = part_val.s
1783	if part_val.quoted:
1784	if eval_flags & QUOTE_FNMATCH:
1785	# [[ foo == /"".py ]] or case (.py) or ${x%.py} or ${x//*.py/}
1786	s = glob_.GlobEscape(s)
1787	elif eval_flags & QUOTE_ERE:
1788	s = glob_.ExtendedRegexEscape(s)
1789	strs.append(s)
1790
1791	elif case(part_value_e.Array):
1792	part_val = cast(part_value.Array, UP_part_val)
1793	if self.exec_opts.strict_array():
1794	# Examples: echo f > "$@"; local foo="$@"
1795
1796	# TODO: This attributes too coarsely, to the word rather than the
1797	# parts. Problem: the word is a TREE of parts, but we only have a
1798	# flat list of part_vals. The only case where we really get arrays
1799	# is "$@", "${a[@]}", "${a[@]//pat/replace}", etc.
1800	e_die(
1801	"This word should yield a string, but it contains an array",
1802	w)
1803
1804	# TODO: Maybe add detail like this.
1805	#e_die('RHS of assignment should only have strings. '
1806	# 'To assign arrays, use b=( "${a[@]}" )')
1807	else:
1808	# It appears to not respect IFS
1809	tmp = [s for s in part_val.strs if s is not None]
1810	s = ' '.join(tmp) # TODO: eliminate double join()?
1811	strs.append(s)
1812
1813	elif case(part_value_e.ExtGlob):
1814	part_val = cast(part_value.ExtGlob, UP_part_val)
1815
1816	# Extended globs are only allowed where we expect them!
1817	if not bool(eval_flags & QUOTE_FNMATCH):
1818	e_die('extended glob not allowed in this word', w)
1819
1820	# recursive call
1821	self._PartValsToString(part_val.part_vals, w, eval_flags,
1822	strs)
1823
1824	else:
1825	raise AssertionError()
1826
1827	def EvalWordToString(self, UP_w, eval_flags=0):
1828	# type: (word_t, int) -> value.Str
1829	"""Given a word, return a string.
1830
1831	Flags can contain a quoting algorithm.
1832	"""
1833	assert UP_w.tag() == word_e.Compound, UP_w
1834	w = cast(CompoundWord, UP_w)
1835
1836	if eval_flags == 0: # QUOTE_FNMATCH etc. breaks optimization
1837	fast_str = word_.FastStrEval(w)
1838	if fast_str is not None:
1839	return value.Str(fast_str)
1840
1841	# Could we additionally optimize a=$b, if we know $b isn't an array
1842	# etc.?
1843
1844	# Note: these empty lists are hot in fib benchmark
1845
1846	part_vals = [] # type: List[part_value_t]
1847	for p in w.parts:
1848	# this doesn't use eval_flags, which is slightly confusing
1849	self._EvalWordPart(p, part_vals, 0)
1850
1851	strs = [] # type: List[str]
1852	self._PartValsToString(part_vals, w, eval_flags, strs)
1853	return value.Str(''.join(strs))
1854
1855	def EvalWordToPattern(self, UP_w):
1856	# type: (rhs_word_t) -> Tuple[value.Str, bool]
1857	"""Like EvalWordToString, but returns whether we got ExtGlob."""
1858	if UP_w.tag() == rhs_word_e.Empty:
1859	return value.Str(''), False
1860
1861	assert UP_w.tag() == rhs_word_e.Compound, UP_w
1862	w = cast(CompoundWord, UP_w)
1863
1864	has_extglob = False
1865	part_vals = [] # type: List[part_value_t]
1866	for p in w.parts:
1867	# this doesn't use eval_flags, which is slightly confusing
1868	self._EvalWordPart(p, part_vals, 0)
1869	if p.tag() == word_part_e.ExtGlob:
1870	has_extglob = True
1871
1872	strs = [] # type: List[str]
1873	self._PartValsToString(part_vals, w, QUOTE_FNMATCH, strs)
1874	return value.Str(''.join(strs)), has_extglob
1875
1876	def EvalForPlugin(self, w):
1877	# type: (CompoundWord) -> value.Str
1878	"""Wrapper around EvalWordToString that prevents errors.
1879
1880	Runtime errors like $(( 1 / 0 )) and mutating $? like $(exit 42)
1881	are handled here.
1882
1883	Similar to ExprEvaluator.PluginCall().
1884	"""
1885	with state.ctx_Registers(self.mem): # to "sandbox" $? and $PIPESTATUS
1886	try:
1887	val = self.EvalWordToString(w)
1888	except error.FatalRuntime as e:
1889	val = value.Str('<Runtime error: %s>' % e.UserErrorString())
1890
1891	except (IOError, OSError) as e:
1892	val = value.Str('<I/O error: %s>' % pyutil.strerror(e))
1893
1894	except KeyboardInterrupt:
1895	val = value.Str('<Ctrl-C>')
1896
1897	return val
1898
1899	def EvalRhsWord(self, UP_w):
1900	# type: (rhs_word_t) -> value_t
1901	"""Used for RHS of assignment.
1902
1903	There is no splitting.
1904	"""
1905	if UP_w.tag() == rhs_word_e.Empty:
1906	return value.Str('')
1907
1908	assert UP_w.tag() == word_e.Compound, UP_w
1909	w = cast(CompoundWord, UP_w)
1910
1911	if len(w.parts) == 1:
1912	part0 = w.parts[0]
1913	UP_part0 = part0
1914	tag = part0.tag()
1915	# Special case for a=(1 2). ShArrayLiteral won't appear in words that
1916	# don't look like assignments.
1917	if tag == word_part_e.ShArrayLiteral:
1918	part0 = cast(ShArrayLiteral, UP_part0)
1919	array_words = part0.words
1920	words = braces.BraceExpandWords(array_words)
1921	strs = self.EvalWordSequence(words)
1922	return value.BashArray(strs)
1923
1924	if tag == word_part_e.BashAssocLiteral:
1925	part0 = cast(word_part.BashAssocLiteral, UP_part0)
1926	d = NewDict() # type: Dict[str, str]
1927	for pair in part0.pairs:
1928	k = self.EvalWordToString(pair.key)
1929	v = self.EvalWordToString(pair.value)
1930	d[k.s] = v.s
1931	return value.BashAssoc(d)
1932
1933	# If RHS doesn't look like a=( ... ), then it must be a string.
1934	return self.EvalWordToString(w)
1935
1936	def _EvalWordFrame(self, frame, argv):
1937	# type: (List[Piece], List[str]) -> None
1938	all_empty = True
1939	all_quoted = True
1940	any_quoted = False
1941
1942	#log('--- frame %s', frame)
1943
1944	for piece in frame:
1945	if len(piece.s):
1946	all_empty = False
1947
1948	if piece.quoted:
1949	any_quoted = True
1950	else:
1951	all_quoted = False
1952
1953	# Elision of ${empty}${empty} but not $empty"$empty" or $empty""
1954	if all_empty and not any_quoted:
1955	return
1956
1957	# If every frag is quoted, e.g. "$a$b" or any part in "${a[@]}"x, then
1958	# don't do word splitting or globbing.
1959	if all_quoted:
1960	tmp = [piece.s for piece in frame]
1961	a = ''.join(tmp)
1962	argv.append(a)
1963	return
1964
1965	will_glob = not self.exec_opts.noglob()
1966
1967	# Array of strings, some of which are BOTH IFS-escaped and GLOB escaped!
1968	frags = [] # type: List[str]
1969	for piece in frame:
1970	if will_glob and piece.quoted:
1971	frag = glob_.GlobEscape(piece.s)
1972	else:
1973	# If we have a literal \, then we turn it into \\\\.
1974	# Splitting takes \\\\ -> \\
1975	# Globbing takes \\ to \ if it doesn't match
1976	frag = _BackslashEscape(piece.s)
1977
1978	if piece.do_split:
1979	frag = _BackslashEscape(frag)
1980	else:
1981	frag = self.splitter.Escape(frag)
1982
1983	frags.append(frag)
1984
1985	flat = ''.join(frags)
1986	#log('flat: %r', flat)
1987
1988	args = self.splitter.SplitForWordEval(flat)
1989
1990	# space=' '; argv $space"". We have a quoted part, but we CANNOT elide.
1991	# Add it back and don't bother globbing.
1992	if len(args) == 0 and any_quoted:
1993	argv.append('')
1994	return
1995
1996	#log('split args: %r', args)
1997	for a in args:
1998	if glob_.LooksLikeGlob(a):
1999	n = self.globber.Expand(a, argv)
2000	if n < 0:
2001	# TODO: location info, with span IDs carried through the frame
2002	raise error.FailGlob('Pattern %r matched no files' % a,
2003	loc.Missing)
2004	else:
2005	argv.append(glob_.GlobUnescape(a))
2006
2007	def _EvalWordToArgv(self, w):
2008	# type: (CompoundWord) -> List[str]
2009	"""Helper for _EvalAssignBuiltin.
2010
2011	Splitting and globbing are disabled for assignment builtins.
2012
2013	Example: declare -"${a[@]}" b=(1 2)
2014	where a is [x b=a d=a]
2015	"""
2016	part_vals = [] # type: List[part_value_t]
2017	self._EvalWordToParts(w, part_vals, 0) # not double quoted
2018	frames = _MakeWordFrames(part_vals)
2019	argv = [] # type: List[str]
2020	for frame in frames:
2021	if len(frame): # empty array gives empty frame!
2022	tmp = [piece.s for piece in frame]
2023	argv.append(''.join(tmp)) # no split or glob
2024	#log('argv: %s', argv)
2025	return argv
2026
2027	def _EvalAssignBuiltin(self, builtin_id, arg0, words):
2028	# type: (builtin_t, str, List[CompoundWord]) -> cmd_value.Assign
2029	"""Handles both static and dynamic assignment, e.g.
2030
2031	x='foo=bar' local a=(1 2) $x
2032	"""
2033	# Grammar:
2034	#
2035	# ('builtin' \| 'command')* keyword flag* pair*
2036	# flag = [-+].*
2037	#
2038	# There is also command -p, but we haven't implemented it. Maybe just punt
2039	# on it. Punted on 'builtin' and 'command' for now too.
2040
2041	eval_to_pairs = True # except for -f and -F
2042	started_pairs = False
2043
2044	flags = [arg0] # initial flags like -p, and -f -F name1 name2
2045	flag_locs = [words[0]]
2046	assign_args = [] # type: List[AssignArg]
2047
2048	n = len(words)
2049	for i in xrange(1, n): # skip first word
2050	w = words[i]
2051
2052	if word_.IsVarLike(w):
2053	started_pairs = True # Everything from now on is an assign_pair
2054
2055	if started_pairs:
2056	left_token, close_token, part_offset = word_.DetectShAssignment(
2057	w)
2058	if left_token: # Detected statically
2059	if left_token.id != Id.Lit_VarLike:
2060	# (not guaranteed since started_pairs is set twice)
2061	e_die('LHS array not allowed in assignment builtin', w)
2062
2063	if lexer.IsPlusEquals(left_token):
2064	var_name = lexer.TokenSliceRight(left_token, -2)
2065	append = True
2066	else:
2067	var_name = lexer.TokenSliceRight(left_token, -1)
2068	append = False
2069
2070	if part_offset == len(w.parts):
2071	rhs = rhs_word.Empty # type: rhs_word_t
2072	else:
2073	# tmp is for intersection of C++/MyPy type systems
2074	tmp = CompoundWord(w.parts[part_offset:])
2075	word_.TildeDetectAssign(tmp)
2076	rhs = tmp
2077
2078	with state.ctx_AssignBuiltin(self.mutable_opts):
2079	right = self.EvalRhsWord(rhs)
2080
2081	arg2 = AssignArg(var_name, right, append, w)
2082	assign_args.append(arg2)
2083
2084	else: # e.g. export $dynamic
2085	argv = self._EvalWordToArgv(w)
2086	for arg in argv:
2087	arg2 = _SplitAssignArg(arg, w)
2088	assign_args.append(arg2)
2089
2090	else:
2091	argv = self._EvalWordToArgv(w)
2092	for arg in argv:
2093	if arg.startswith('-') or arg.startswith('+'):
2094	# e.g. declare -r +r
2095	flags.append(arg)
2096	flag_locs.append(w)
2097
2098	# Shortcut that relies on -f and -F always meaning "function" for
2099	# all assignment builtins
2100	if 'f' in arg or 'F' in arg:
2101	eval_to_pairs = False
2102
2103	else: # e.g. export $dynamic
2104	if eval_to_pairs:
2105	arg2 = _SplitAssignArg(arg, w)
2106	assign_args.append(arg2)
2107	started_pairs = True
2108	else:
2109	flags.append(arg)
2110
2111	return cmd_value.Assign(builtin_id, flags, flag_locs, assign_args)
2112
2113	def SimpleEvalWordSequence2(self, words, allow_assign):
2114	# type: (List[CompoundWord], bool) -> cmd_value_t
2115	"""Simple word evaluation for YSH."""
2116	strs = [] # type: List[str]
2117	locs = [] # type: List[CompoundWord]
2118
2119	for i, w in enumerate(words):
2120	# No globbing in the first arg for command.Simple.
2121	if i == 0 and allow_assign:
2122	strs0 = self._EvalWordToArgv(w) # respects strict-array
2123	if len(strs0) == 1:
2124	arg0 = strs0[0]
2125	builtin_id = consts.LookupAssignBuiltin(arg0)
2126	if builtin_id != consts.NO_INDEX:
2127	# Same logic as legacy word eval, with no splitting
2128	return self._EvalAssignBuiltin(builtin_id, arg0, words)
2129
2130	strs.extend(strs0)
2131	for _ in strs0:
2132	locs.append(w)
2133	continue
2134
2135	if glob_.LooksLikeStaticGlob(w):
2136	val = self.EvalWordToString(w) # respects strict-array
2137	num_appended = self.globber.Expand(val.s, strs)
2138	if num_appended < 0:
2139	raise error.FailGlob('Pattern %r matched no files' % val.s,
2140	w)
2141	for _ in xrange(num_appended):
2142	locs.append(w)
2143	continue
2144
2145	part_vals = [] # type: List[part_value_t]
2146	self._EvalWordToParts(w, part_vals, 0) # not quoted
2147
2148	if 0:
2149	log('')
2150	log('Static: part_vals after _EvalWordToParts:')
2151	for entry in part_vals:
2152	log(' %s', entry)
2153
2154	# Still need to process
2155	frames = _MakeWordFrames(part_vals)
2156
2157	if 0:
2158	log('')
2159	log('Static: frames after _MakeWordFrames:')
2160	for entry in frames:
2161	log(' %s', entry)
2162
2163	# We will still allow x"${a[@]"x, though it's deprecated by @a, which
2164	# disallows such expressions at parse time.
2165	for frame in frames:
2166	if len(frame): # empty array gives empty frame!
2167	tmp = [piece.s for piece in frame]
2168	strs.append(''.join(tmp)) # no split or glob
2169	locs.append(w)
2170
2171	return cmd_value.Argv(strs, locs, None, None, None, None)
2172
2173	def EvalWordSequence2(self, words, allow_assign=False):
2174	# type: (List[CompoundWord], bool) -> cmd_value_t
2175	"""Turns a list of Words into a list of strings.
2176
2177	Unlike the EvalWord*() methods, it does globbing.
2178
2179	Args:
2180	words: list of Word instances
2181
2182	Returns:
2183	argv: list of string arguments, or None if there was an eval error
2184	"""
2185	if self.exec_opts.simple_word_eval():
2186	return self.SimpleEvalWordSequence2(words, allow_assign)
2187
2188	# Parse time:
2189	# 1. brace expansion. TODO: Do at parse time.
2190	# 2. Tilde detection. DONE at parse time. Only if Id.Lit_Tilde is the
2191	# first WordPart.
2192	#
2193	# Run time:
2194	# 3. tilde sub, var sub, command sub, arith sub. These are all
2195	# "concurrent" on WordParts. (optional process sub with <() )
2196	# 4. word splitting. Can turn this off with a shell option? Definitely
2197	# off for oil.
2198	# 5. globbing -- several exec_opts affect this: nullglob, safeglob, etc.
2199
2200	#log('W %s', words)
2201	strs = [] # type: List[str]
2202	locs = [] # type: List[CompoundWord]
2203
2204	n = 0
2205	for i, w in enumerate(words):
2206	fast_str = word_.FastStrEval(w)
2207	if fast_str is not None:
2208	strs.append(fast_str)
2209	locs.append(w)
2210
2211	# e.g. the 'local' in 'local a=b c=d' will be here
2212	if allow_assign and i == 0:
2213	builtin_id = consts.LookupAssignBuiltin(fast_str)
2214	if builtin_id != consts.NO_INDEX:
2215	return self._EvalAssignBuiltin(builtin_id, fast_str,
2216	words)
2217	continue
2218
2219	part_vals = [] # type: List[part_value_t]
2220	self._EvalWordToParts(w, part_vals, EXTGLOB_FILES)
2221
2222	# DYNAMICALLY detect if we're going to run an assignment builtin, and
2223	# change the rest of the evaluation algorithm if so.
2224	#
2225	# We want to allow:
2226	# e=export
2227	# $e foo=bar
2228	#
2229	# But we don't want to evaluate the first word twice in the case of:
2230	# $(some-command) --flag
2231	if allow_assign and i == 0 and len(part_vals) == 1:
2232	val0 = part_vals[0]
2233	UP_val0 = val0
2234	if val0.tag() == part_value_e.String:
2235	val0 = cast(Piece, UP_val0)
2236	if not val0.quoted:
2237	builtin_id = consts.LookupAssignBuiltin(val0.s)
2238	if builtin_id != consts.NO_INDEX:
2239	return self._EvalAssignBuiltin(
2240	builtin_id, val0.s, words)
2241
2242	if 0:
2243	log('')
2244	log('part_vals after _EvalWordToParts:')
2245	for entry in part_vals:
2246	log(' %s', entry)
2247
2248	frames = _MakeWordFrames(part_vals)
2249	if 0:
2250	log('')
2251	log('frames after _MakeWordFrames:')
2252	for entry in frames:
2253	log(' %s', entry)
2254
2255	# Do splitting and globbing. Each frame will append zero or more args.
2256	for frame in frames:
2257	self._EvalWordFrame(frame, strs)
2258
2259	# Fill in locations parallel to strs.
2260	n_next = len(strs)
2261	for _ in xrange(n_next - n):
2262	locs.append(w)
2263	n = n_next
2264
2265	# A non-assignment command.
2266	# NOTE: Can't look up builtins here like we did for assignment, because
2267	# functions can override builtins.
2268	return cmd_value.Argv(strs, locs, None, None, None, None)
2269
2270	def EvalWordSequence(self, words):
2271	# type: (List[CompoundWord]) -> List[str]
2272	"""For arrays and for loops.
2273
2274	They don't allow assignment builtins.
2275	"""
2276	UP_cmd_val = self.EvalWordSequence2(words)
2277
2278	assert UP_cmd_val.tag() == cmd_value_e.Argv
2279	cmd_val = cast(cmd_value.Argv, UP_cmd_val)
2280	return cmd_val.argv
2281
2282
2283	class NormalWordEvaluator(AbstractWordEvaluator):
2284
2285	def __init__(
2286	self,
2287	mem, # type: state.Mem
2288	exec_opts, # type: optview.Exec
2289	mutable_opts, # type: state.MutableOpts
2290	tilde_ev, # type: TildeEvaluator
2291	splitter, # type: SplitContext
2292	errfmt, # type: ErrorFormatter
2293	):
2294	# type: (...) -> None
2295	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2296	tilde_ev, splitter, errfmt)
2297	self.shell_ex = None # type: _Executor
2298
2299	def CheckCircularDeps(self):
2300	# type: () -> None
2301	assert self.arith_ev is not None
2302	# Disabled for pure OSH
2303	#assert self.expr_ev is not None
2304	assert self.shell_ex is not None
2305	assert self.prompt_ev is not None
2306
2307	def _EvalCommandSub(self, cs_part, quoted):
2308	# type: (CommandSub, bool) -> part_value_t
2309	stdout_str = self.shell_ex.RunCommandSub(cs_part)
2310
2311	if cs_part.left_token.id == Id.Left_AtParen:
2312	# YSH splitting algorithm: does not depend on IFS
2313	try:
2314	strs = j8.SplitJ8Lines(stdout_str)
2315	except error.Decode as e:
2316	# status code 4 is special, for encode/decode errors.
2317	raise error.Structured(4, e.Message(), cs_part.left_token)
2318
2319	#strs = self.splitter.SplitForWordEval(stdout_str)
2320	return part_value.Array(strs)
2321	else:
2322	return Piece(stdout_str, quoted, not quoted)
2323
2324	def _EvalProcessSub(self, cs_part):
2325	# type: (CommandSub) -> Piece
2326	dev_path = self.shell_ex.RunProcessSub(cs_part)
2327	# pretend it's quoted; no split or glob
2328	return Piece(dev_path, True, False)
2329
2330
2331	_DUMMY = '__NO_COMMAND_SUB__'
2332
2333
2334	class CompletionWordEvaluator(AbstractWordEvaluator):
2335	"""An evaluator that has no access to an executor.
2336
2337	NOTE: core/completion.py doesn't actually try to use these strings to
2338	complete. If you have something like 'echo $(echo hi)/f<TAB>', it sees the
2339	inner command as the last one, and knows that it is not at the end of the
2340	line.
2341	"""
2342
2343	def __init__(
2344	self,
2345	mem, # type: state.Mem
2346	exec_opts, # type: optview.Exec
2347	mutable_opts, # type: state.MutableOpts
2348	tilde_ev, # type: TildeEvaluator
2349	splitter, # type: SplitContext
2350	errfmt, # type: ErrorFormatter
2351	):
2352	# type: (...) -> None
2353	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2354	tilde_ev, splitter, errfmt)
2355
2356	def CheckCircularDeps(self):
2357	# type: () -> None
2358	assert self.prompt_ev is not None
2359	assert self.arith_ev is not None
2360	assert self.expr_ev is not None
2361
2362	def _EvalCommandSub(self, cs_part, quoted):
2363	# type: (CommandSub, bool) -> part_value_t
2364	if cs_part.left_token.id == Id.Left_AtParen:
2365	return part_value.Array([_DUMMY])
2366	else:
2367	return Piece(_DUMMY, quoted, not quoted)
2368
2369	def _EvalProcessSub(self, cs_part):
2370	# type: (CommandSub) -> Piece
2371	# pretend it's quoted; no split or glob
2372	return Piece('__NO_PROCESS_SUB__', True, False)
2373
2374
2375	# vim: sw=4