osh/word_eval.py

OILS / osh / word_eval.py View on Github | oilshell.org

2374 lines, 1433 significant

1	"""
2	word_eval.py - Evaluator for the word language.
3	"""
4
5	from _devbuild.gen.id_kind_asdl import Id, Kind, Kind_str
6	from _devbuild.gen.syntax_asdl import (
7	Token,
8	SimpleVarSub,
9	loc,
10	loc_t,
11	BracedVarSub,
12	CommandSub,
13	bracket_op,
14	bracket_op_e,
15	suffix_op,
16	suffix_op_e,
17	ShArrayLiteral,
18	SingleQuoted,
19	DoubleQuoted,
20	word_e,
21	word_t,
22	CompoundWord,
23	rhs_word,
24	rhs_word_e,
25	rhs_word_t,
26	word_part,
27	word_part_e,
28	)
29	from _devbuild.gen.runtime_asdl import (
30	part_value,
31	part_value_e,
32	part_value_t,
33	cmd_value,
34	cmd_value_e,
35	cmd_value_t,
36	AssignArg,
37	a_index,
38	a_index_e,
39	VTestPlace,
40	VarSubState,
41	Piece,
42	)
43	from _devbuild.gen.option_asdl import option_i
44	from _devbuild.gen.value_asdl import (
45	value,
46	value_e,
47	value_t,
48	sh_lvalue,
49	sh_lvalue_t,
50	)
51	from core import error
52	from core import pyos
53	from core import pyutil
54	from core import state
55	from core import ui
56	from core import util
57	from data_lang import j8
58	from data_lang import j8_lite
59	from core.error import e_die
60	from frontend import consts
61	from frontend import lexer
62	from frontend import location
63	from mycpp import mops
64	from mycpp.mylib import log, tagswitch, NewDict
65	from osh import braces
66	from osh import glob_
67	from osh import string_ops
68	from osh import word_
69	from ysh import expr_eval
70	from ysh import val_ops
71
72	from typing import Optional, Tuple, List, Dict, cast, TYPE_CHECKING
73
74	if TYPE_CHECKING:
75	from _devbuild.gen.syntax_asdl import word_part_t
76	from _devbuild.gen.option_asdl import builtin_t
77	from core import optview
78	from core.state import Mem
79	from core.ui import ErrorFormatter
80	from core.vm import _Executor
81	from osh.split import SplitContext
82	from osh import prompt
83	from osh import sh_expr_eval
84
85	# Flags for _EvalWordToParts and _EvalWordPart (not all are used for both)
86	QUOTED = 1 << 0
87	IS_SUBST = 1 << 1
88
89	EXTGLOB_FILES = 1 << 2 # allow @(cc) from file system?
90	EXTGLOB_MATCH = 1 << 3 # allow @(cc) in pattern matching?
91	EXTGLOB_NESTED = 1 << 4 # for @(one\|!(two\|three))
92
93	# For EvalWordToString
94	QUOTE_FNMATCH = 1 << 5
95	QUOTE_ERE = 1 << 6
96
97	# For compatibility, ${BASH_SOURCE} and ${BASH_SOURCE[@]} are both valid.
98	# Ditto for ${FUNCNAME} and ${BASH_LINENO}.
99	_STRING_AND_ARRAY = ['BASH_SOURCE', 'FUNCNAME', 'BASH_LINENO']
100
101
102	def ShouldArrayDecay(var_name, exec_opts, is_plain_var_sub=True):
103	# type: (str, optview.Exec, bool) -> bool
104	"""Return whether we should allow ${a} to mean ${a[0]}."""
105	return (not exec_opts.strict_array() or
106	is_plain_var_sub and var_name in _STRING_AND_ARRAY)
107
108
109	def DecayArray(val):
110	# type: (value_t) -> value_t
111	"""Resolve ${array} to ${array[0]}."""
112	if val.tag() == value_e.BashArray:
113	array_val = cast(value.BashArray, val)
114	s = array_val.strs[0] if len(array_val.strs) else None
115	elif val.tag() == value_e.BashAssoc:
116	assoc_val = cast(value.BashAssoc, val)
117	s = assoc_val.d['0'] if '0' in assoc_val.d else None
118	else:
119	raise AssertionError(val.tag())
120
121	if s is None:
122	return value.Undef
123	else:
124	return value.Str(s)
125
126
127	def GetArrayItem(strs, index):
128	# type: (List[str], int) -> Optional[str]
129
130	n = len(strs)
131	if index < 0:
132	index += n
133
134	if 0 <= index and index < n:
135	# TODO: strs->index() has a redundant check for (i < 0)
136	s = strs[index]
137	# note: s could be None because representation is sparse
138	else:
139	s = None
140	return s
141
142
143	# Use libc to parse NAME, NAME=value, and NAME+=value. We want submatch
144	# extraction, but I haven't used that in re2c, and we would need a new kind of
145	# binding.
146	#
147	ASSIGN_ARG_RE = '^([a-zA-Z_][a-zA-Z0-9_])((=\|\+=)(.))?$'
148
149	# Eggex equivalent:
150	#
151	# VarName = /
152	# [a-z A-Z _ ]
153	# [a-z A-Z 0-9 _ ]*
154	# /
155	#
156	# SplitArg = /
157	# %begin
158	# < VarName >
159	# < < '=' \| '+=' > < dot* > > ?
160	# %end
161	# /
162	# Note: must use < > for grouping because there is no non-capturing group.
163
164
165	def _SplitAssignArg(arg, blame_word):
166	# type: (str, CompoundWord) -> AssignArg
167	"""Dynamically parse argument to declare, export, etc.
168
169	This is a fallback to the static parsing done below.
170	"""
171	# Note: it would be better to cache regcomp(), but we don't have an API for
172	# that, and it probably isn't a bottleneck now
173	m = util.RegexSearch(ASSIGN_ARG_RE, arg)
174	if m is None:
175	e_die("Assignment builtin expected NAME=value, got %r" % arg,
176	blame_word)
177
178	var_name = m[1]
179	# m[2] is used for grouping; ERE doesn't have non-capturing groups
180
181	op = m[3]
182	assert op is not None, op
183	if len(op): # declare NAME=
184	val = value.Str(m[4]) # type: Optional[value_t]
185	append = op[0] == '+'
186	else: # declare NAME
187	val = None # no operator
188	append = False
189
190	return AssignArg(var_name, val, append, blame_word)
191
192
193	# NOTE: Could be done with util.BackslashEscape like glob_.GlobEscape().
194	def _BackslashEscape(s):
195	# type: (str) -> str
196	"""Double up backslashes.
197
198	Useful for strings about to be globbed and strings about to be IFS
199	escaped.
200	"""
201	return s.replace('\\', '\\\\')
202
203
204	def _ValueToPartValue(val, quoted, part_loc):
205	# type: (value_t, bool, word_part_t) -> part_value_t
206	"""Helper for VarSub evaluation.
207
208	Called by _EvalBracedVarSub and _EvalWordPart for SimpleVarSub.
209	"""
210	UP_val = val
211
212	with tagswitch(val) as case:
213	if case(value_e.Undef):
214	# This happens in the case of ${undef+foo}. We skipped _EmptyStrOrError,
215	# but we have to append to the empty string.
216	return Piece('', quoted, not quoted)
217
218	elif case(value_e.Str):
219	val = cast(value.Str, UP_val)
220	return Piece(val.s, quoted, not quoted)
221
222	elif case(value_e.BashArray):
223	val = cast(value.BashArray, UP_val)
224	return part_value.Array(val.strs)
225
226	elif case(value_e.BashAssoc):
227	val = cast(value.BashAssoc, UP_val)
228	# bash behavior: splice values!
229	return part_value.Array(val.d.values())
230
231	# Cases added for YSH
232	# value_e.List is also here - we use val_ops.stringify()s err message
233	elif case(value_e.Null, value_e.Bool, value_e.Int, value_e.Float,
234	value_e.Eggex, value_e.List):
235	s = val_ops.Stringify(val, loc.Missing)
236	return Piece(s, quoted, not quoted)
237
238	else:
239	raise error.TypeErr(val, "Can't substitute into word",
240	loc.WordPart(part_loc))
241
242	raise AssertionError('for -Wreturn-type in C++')
243
244
245	def _MakeWordFrames(part_vals):
246	# type: (List[part_value_t]) -> List[List[Piece]]
247	"""A word evaluates to a flat list of part_value (String or Array). frame
248	is a portion that results in zero or more args. It can never be joined.
249	This idea exists because of arrays like "$@" and "${a[@]}".
250
251	Example:
252
253	a=(1 '2 3' 4)
254	x=x
255	y=y
256
257	# This word
258	$x"${a[@]}"$y
259
260	# Results in Three frames:
261	[ ('x', False, True), ('1', True, False) ]
262	[ ('2 3', True, False) ]
263	[ ('4', True, False), ('y', False, True) ]
264
265	Note: A frame is a 3-tuple that's identical to Piece()? Maybe we
266	should make that top level type.
267
268	TODO:
269	- Instead of List[List[Piece]], where List[Piece] is a Frame
270	- Change this representation to
271	Frames = (List[Piece] pieces, List[int] break_indices)
272	# where break_indices are the end
273
274	Consider a common case like "$x" or "${x}" - I think this a lot more
275	efficient?
276
277	And then change _EvalWordFrame(pieces: List[Piece], start: int, end: int)
278	"""
279	current = [] # type: List[Piece]
280	frames = [current]
281
282	for p in part_vals:
283	UP_p = p
284
285	with tagswitch(p) as case:
286	if case(part_value_e.String):
287	p = cast(Piece, UP_p)
288	current.append(p)
289
290	elif case(part_value_e.Array):
291	p = cast(part_value.Array, UP_p)
292
293	is_first = True
294	for s in p.strs:
295	if s is None:
296	continue # ignore undefined array entries
297
298	# Arrays parts are always quoted; otherwise they would have decayed to
299	# a string.
300	piece = Piece(s, True, False)
301	if is_first:
302	current.append(piece)
303	is_first = False
304	else:
305	current = [piece]
306	frames.append(current) # singleton frame
307
308	else:
309	raise AssertionError()
310
311	return frames
312
313
314	# TODO: This could be _MakeWordFrames and then sep.join(). It's redundant.
315	def _DecayPartValuesToString(part_vals, join_char):
316	# type: (List[part_value_t], str) -> str
317	# Decay ${a=x"$@"x} to string.
318	out = [] # type: List[str]
319	for p in part_vals:
320	UP_p = p
321	with tagswitch(p) as case:
322	if case(part_value_e.String):
323	p = cast(Piece, UP_p)
324	out.append(p.s)
325	elif case(part_value_e.Array):
326	p = cast(part_value.Array, UP_p)
327	# TODO: Eliminate double join for speed?
328	tmp = [s for s in p.strs if s is not None]
329	out.append(join_char.join(tmp))
330	else:
331	raise AssertionError()
332	return ''.join(out)
333
334
335	def _PerformSlice(
336	val, # type: value_t
337	begin, # type: int
338	length, # type: int
339	has_length, # type: bool
340	part, # type: BracedVarSub
341	arg0_val, # type: value.Str
342	):
343	# type: (...) -> value_t
344	UP_val = val
345	with tagswitch(val) as case:
346	if case(value_e.Str): # Slice UTF-8 characters in a string.
347	val = cast(value.Str, UP_val)
348	s = val.s
349	n = len(s)
350
351	if begin < 0: # Compute offset with unicode
352	byte_begin = n
353	num_iters = -begin
354	for _ in xrange(num_iters):
355	byte_begin = string_ops.PreviousUtf8Char(s, byte_begin)
356	else:
357	byte_begin = string_ops.AdvanceUtf8Chars(s, begin, 0)
358
359	if has_length:
360	if length < 0: # Compute offset with unicode
361	# Confusing: this is a POSITION
362	byte_end = n
363	num_iters = -length
364	for _ in xrange(num_iters):
365	byte_end = string_ops.PreviousUtf8Char(s, byte_end)
366	else:
367	byte_end = string_ops.AdvanceUtf8Chars(
368	s, length, byte_begin)
369	else:
370	byte_end = len(s)
371
372	substr = s[byte_begin:byte_end]
373	result = value.Str(substr) # type: value_t
374
375	elif case(value_e.BashArray): # Slice array entries.
376	val = cast(value.BashArray, UP_val)
377	# NOTE: This error is ALWAYS fatal in bash. It's inconsistent with
378	# strings.
379	if has_length and length < 0:
380	e_die(
381	"The length index of a array slice can't be negative: %d" %
382	length, loc.WordPart(part))
383
384	# Quirk: "begin" for positional arguments ($@ and $*) counts $0.
385	if arg0_val is not None:
386	orig = [arg0_val.s]
387	orig.extend(val.strs)
388	else:
389	orig = val.strs
390
391	n = len(orig)
392	if begin < 0:
393	i = n + begin # ${@:-3} starts counts from the end
394	else:
395	i = begin
396	strs = [] # type: List[str]
397	count = 0
398	while i < n:
399	if has_length and count == length: # length could be 0
400	break
401	s = orig[i]
402	if s is not None: # Unset elements don't count towards the length
403	strs.append(s)
404	count += 1
405	i += 1
406
407	result = value.BashArray(strs)
408
409	elif case(value_e.BashAssoc):
410	e_die("Can't slice associative arrays", loc.WordPart(part))
411
412	else:
413	raise error.TypeErr(val, 'Slice op expected Str or BashArray',
414	loc.WordPart(part))
415
416	return result
417
418
419	class StringWordEvaluator(object):
420	"""Interface used by ArithEvaluator / BoolEvaluator"""
421
422	def __init__(self):
423	# type: () -> None
424	"""Empty constructor for mycpp."""
425	pass
426
427	def EvalWordToString(self, w, eval_flags=0):
428	# type: (word_t, int) -> value.Str
429	raise NotImplementedError()
430
431
432	def _GetDollarHyphen(exec_opts):
433	# type: (optview.Exec) -> str
434	chars = [] # type: List[str]
435	if exec_opts.interactive():
436	chars.append('i')
437
438	if exec_opts.errexit():
439	chars.append('e')
440	if exec_opts.noglob():
441	chars.append('f')
442	if exec_opts.noexec():
443	chars.append('n')
444	if exec_opts.nounset():
445	chars.append('u')
446	# NO letter for pipefail?
447	if exec_opts.xtrace():
448	chars.append('x')
449	if exec_opts.noclobber():
450	chars.append('C')
451
452	# bash has:
453	# - c for sh -c, i for sh -i (mksh also has this)
454	# - h for hashing (mksh also has this)
455	# - B for brace expansion
456	return ''.join(chars)
457
458
459	class TildeEvaluator(object):
460
461	def __init__(self, mem, exec_opts):
462	# type: (Mem, optview.Exec) -> None
463	self.mem = mem
464	self.exec_opts = exec_opts
465
466	def GetMyHomeDir(self):
467	# type: () -> Optional[str]
468	"""Consult $HOME first, and then make a libc call.
469
470	Important: the libc call can FAIL, which is why we prefer $HOME. See issue
471	#1578.
472	"""
473	# First look up the HOME var, then ask the OS. This is what bash does.
474	val = self.mem.GetValue('HOME')
475	UP_val = val
476	if val.tag() == value_e.Str:
477	val = cast(value.Str, UP_val)
478	return val.s
479	return pyos.GetMyHomeDir()
480
481	def Eval(self, part):
482	# type: (word_part.TildeSub) -> str
483	"""Evaluates ~ and ~user, given a Lit_TildeLike token."""
484
485	if part.user_name is None:
486	result = self.GetMyHomeDir()
487	else:
488	result = pyos.GetHomeDir(part.user_name)
489
490	if result is None:
491	if self.exec_opts.strict_tilde():
492	e_die("Error expanding tilde (e.g. invalid user)", part.left)
493	else:
494	# Return ~ or ~user literally
495	result = '~'
496	if part.user_name is not None:
497	result = result + part.user_name # mycpp doesn't have +=
498
499	return result
500
501
502	class AbstractWordEvaluator(StringWordEvaluator):
503	"""Abstract base class for word evaluators.
504
505	Public entry points:
506	EvalWordToString EvalForPlugin EvalRhsWord
507	EvalWordSequence EvalWordSequence2
508	"""
509
510	def __init__(
511	self,
512	mem, # type: state.Mem
513	exec_opts, # type: optview.Exec
514	mutable_opts, # type: state.MutableOpts
515	tilde_ev, # type: TildeEvaluator
516	splitter, # type: SplitContext
517	errfmt, # type: ui.ErrorFormatter
518	):
519	# type: (...) -> None
520	self.arith_ev = None # type: sh_expr_eval.ArithEvaluator
521	self.expr_ev = None # type: expr_eval.ExprEvaluator
522	self.prompt_ev = None # type: prompt.Evaluator
523
524	self.unsafe_arith = None # type: sh_expr_eval.UnsafeArith
525
526	self.tilde_ev = tilde_ev
527
528	self.mem = mem # for $HOME, $1, etc.
529	self.exec_opts = exec_opts # for nounset
530	self.mutable_opts = mutable_opts # for _allow_command_sub
531	self.splitter = splitter
532	self.errfmt = errfmt
533
534	self.globber = glob_.Globber(exec_opts)
535
536	def CheckCircularDeps(self):
537	# type: () -> None
538	raise NotImplementedError()
539
540	def _EvalCommandSub(self, cs_part, quoted):
541	# type: (CommandSub, bool) -> part_value_t
542	"""Abstract since it has a side effect."""
543	raise NotImplementedError()
544
545	def _EvalProcessSub(self, cs_part):
546	# type: (CommandSub) -> part_value_t
547	"""Abstract since it has a side effect."""
548	raise NotImplementedError()
549
550	def _EvalVarNum(self, var_num):
551	# type: (int) -> value_t
552	assert var_num >= 0
553	return self.mem.GetArgNum(var_num)
554
555	def _EvalSpecialVar(self, op_id, quoted, vsub_state):
556	# type: (int, bool, VarSubState) -> value_t
557	"""Evaluate $?
558
559	and so forth
560	"""
561	# $@ is special -- it need to know whether it is in a double quoted
562	# context.
563	#
564	# - If it's $@ in a double quoted context, return an ARRAY.
565	# - If it's $@ in a normal context, return a STRING, which then will be
566	# subject to splitting.
567
568	if op_id in (Id.VSub_At, Id.VSub_Star):
569	argv = self.mem.GetArgv()
570	val = value.BashArray(argv) # type: value_t
571	if op_id == Id.VSub_At:
572	# "$@" evaluates to an array, $@ should be decayed
573	vsub_state.join_array = not quoted
574	else: # $* "$*" are both decayed
575	vsub_state.join_array = True
576
577	elif op_id == Id.VSub_Hyphen:
578	val = value.Str(_GetDollarHyphen(self.exec_opts))
579
580	else:
581	val = self.mem.GetSpecialVar(op_id)
582
583	return val
584
585	def _ApplyTestOp(
586	self,
587	val, # type: value_t
588	op, # type: suffix_op.Unary
589	quoted, # type: bool
590	part_vals, # type: Optional[List[part_value_t]]
591	vtest_place, # type: VTestPlace
592	blame_token, # type: Token
593	):
594	# type: (...) -> bool
595	"""
596	Returns:
597	Whether part_vals was mutated
598
599	${a:-} returns part_value[]
600	${a:+} returns part_value[]
601	${a:?error} returns error word?
602	${a:=} returns part_value[] but also needs self.mem for side effects.
603
604	So I guess it should return part_value[], and then a flag for raising an
605	error, and then a flag for assigning it?
606	The original BracedVarSub will have the name.
607
608	Example of needing multiple part_value[]
609
610	echo X-${a:-'def'"ault"}-X
611
612	We return two part values from the BracedVarSub. Also consider:
613
614	echo ${a:-x"$@"x}
615	"""
616	eval_flags = IS_SUBST
617	if quoted:
618	eval_flags \|= QUOTED
619
620	tok = op.op
621	# NOTE: Splicing part_values is necessary because of code like
622	# ${undef:-'a b' c 'd # e'}. Each part_value can have a different
623	# do_glob/do_elide setting.
624	UP_val = val
625	with tagswitch(val) as case:
626	if case(value_e.Undef):
627	is_falsey = True
628	elif case(value_e.Str):
629	val = cast(value.Str, UP_val)
630	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_ColonEquals,
631	Id.VTest_ColonQMark, Id.VTest_ColonPlus):
632	is_falsey = len(val.s) == 0
633	else:
634	is_falsey = False
635	elif case(value_e.BashArray):
636	val = cast(value.BashArray, UP_val)
637	# TODO: allow undefined
638	is_falsey = len(val.strs) == 0
639	elif case(value_e.BashAssoc):
640	val = cast(value.BashAssoc, UP_val)
641	is_falsey = len(val.d) == 0
642	else:
643	# value.Eggex, etc. are all false
644	is_falsey = False
645
646	if tok.id in (Id.VTest_ColonHyphen, Id.VTest_Hyphen):
647	if is_falsey:
648	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
649	return True
650	else:
651	return False
652
653	# Inverse of the above.
654	elif tok.id in (Id.VTest_ColonPlus, Id.VTest_Plus):
655	if is_falsey:
656	return False
657	else:
658	self._EvalRhsWordToParts(op.arg_word, part_vals, eval_flags)
659	return True
660
661	# Splice and assign
662	elif tok.id in (Id.VTest_ColonEquals, Id.VTest_Equals):
663	if is_falsey:
664	# Collect new part vals.
665	assign_part_vals = [] # type: List[part_value_t]
666	self._EvalRhsWordToParts(op.arg_word, assign_part_vals,
667	eval_flags)
668	# Append them to out param AND return them.
669	part_vals.extend(assign_part_vals)
670
671	if vtest_place.name is None:
672	# TODO: error context
673	e_die("Can't assign to special variable")
674	else:
675	# NOTE: This decays arrays too! 'shopt -s strict_array' could
676	# avoid it.
677	rhs_str = _DecayPartValuesToString(
678	assign_part_vals, self.splitter.GetJoinChar())
679	if vtest_place.index is None: # using None when no index
680	lval = location.LName(
681	vtest_place.name) # type: sh_lvalue_t
682	else:
683	var_name = vtest_place.name
684	var_index = vtest_place.index
685	UP_var_index = var_index
686
687	with tagswitch(var_index) as case:
688	if case(a_index_e.Int):
689	var_index = cast(a_index.Int, UP_var_index)
690	lval = sh_lvalue.Indexed(
691	var_name, var_index.i, loc.Missing)
692	elif case(a_index_e.Str):
693	var_index = cast(a_index.Str, UP_var_index)
694	lval = sh_lvalue.Keyed(var_name, var_index.s,
695	loc.Missing)
696	else:
697	raise AssertionError()
698
699	state.OshLanguageSetValue(self.mem, lval,
700	value.Str(rhs_str))
701	return True
702
703	else:
704	return False
705
706	elif tok.id in (Id.VTest_ColonQMark, Id.VTest_QMark):
707	if is_falsey:
708	# The arg is the error message
709	error_part_vals = [] # type: List[part_value_t]
710	self._EvalRhsWordToParts(op.arg_word, error_part_vals,
711	eval_flags)
712	error_str = _DecayPartValuesToString(
713	error_part_vals, self.splitter.GetJoinChar())
714	e_die("unset variable %r" % error_str, blame_token)
715
716	else:
717	return False
718
719	else:
720	raise AssertionError(tok.id)
721
722	def _Length(self, val, token):
723	# type: (value_t, Token) -> int
724	"""Returns the length of the value, for ${#var}"""
725	UP_val = val
726	with tagswitch(val) as case:
727	if case(value_e.Str):
728	val = cast(value.Str, UP_val)
729	# NOTE: Whether bash counts bytes or chars is affected by LANG
730	# environment variables.
731	# Should we respect that, or another way to select? set -o
732	# count-bytes?
733
734	# https://stackoverflow.com/questions/17368067/length-of-string-in-bash
735	try:
736	length = string_ops.CountUtf8Chars(val.s)
737	except error.Strict as e:
738	# Add this here so we don't have to add it so far down the stack.
739	# TODO: It's better to show BOTH this CODE an the actual DATA
740	# somehow.
741	e.location = token
742
743	if self.exec_opts.strict_word_eval():
744	raise
745	else:
746	# NOTE: Doesn't make the command exit with 1; it just returns a
747	# length of -1.
748	self.errfmt.PrettyPrintError(e, prefix='warning: ')
749	return -1
750
751	elif case(value_e.BashArray):
752	val = cast(value.BashArray, UP_val)
753	# There can be empty placeholder values in the array.
754	length = 0
755	for s in val.strs:
756	if s is not None:
757	length += 1
758
759	elif case(value_e.BashAssoc):
760	val = cast(value.BashAssoc, UP_val)
761	length = len(val.d)
762
763	else:
764	raise error.TypeErr(
765	val, "Length op expected Str, BashArray, BashAssoc", token)
766
767	return length
768
769	def _Keys(self, val, token):
770	# type: (value_t, Token) -> value_t
771	"""Return keys of a container, for ${!array[@]}"""
772
773	UP_val = val
774	with tagswitch(val) as case:
775	if case(value_e.BashArray):
776	val = cast(value.BashArray, UP_val)
777	# translation issue: tuple indices not supported in list comprehensions
778	#indices = [str(i) for i, s in enumerate(val.strs) if s is not None]
779	indices = [] # type: List[str]
780	for i, s in enumerate(val.strs):
781	if s is not None:
782	indices.append(str(i))
783	return value.BashArray(indices)
784
785	elif case(value_e.BashAssoc):
786	val = cast(value.BashAssoc, UP_val)
787	assert val.d is not None # for MyPy, so it's not Optional[]
788
789	# BUG: Keys aren't ordered according to insertion!
790	return value.BashArray(val.d.keys())
791
792	else:
793	raise error.TypeErr(val, 'Keys op expected Str', token)
794
795	def _EvalVarRef(self, val, blame_tok, quoted, vsub_state, vtest_place):
796	# type: (value_t, Token, bool, VarSubState, VTestPlace) -> value_t
797	"""Handles indirect expansion like ${!var} and ${!a[0]}.
798
799	Args:
800	blame_tok: 'foo' for ${!foo}
801	"""
802	UP_val = val
803	with tagswitch(val) as case:
804	if case(value_e.Undef):
805	return value.Undef # ${!undef} is just weird bash behavior
806
807	elif case(value_e.Str):
808	val = cast(value.Str, UP_val)
809	bvs_part = self.unsafe_arith.ParseVarRef(val.s, blame_tok)
810	return self._VarRefValue(bvs_part, quoted, vsub_state,
811	vtest_place)
812
813	elif case(value_e.BashArray): # caught earlier but OK
814	e_die('Indirect expansion of array')
815
816	elif case(value_e.BashAssoc): # caught earlier but OK
817	e_die('Indirect expansion of assoc array')
818
819	else:
820	raise error.TypeErr(val, 'Var Ref op expected Str', blame_tok)
821
822	def _ApplyUnarySuffixOp(self, val, op):
823	# type: (value_t, suffix_op.Unary) -> value_t
824	assert val.tag() != value_e.Undef
825
826	op_kind = consts.GetKind(op.op.id)
827
828	if op_kind == Kind.VOp1:
829	# NOTE: glob syntax is supported in ^ ^^ , ,, ! As well as % %% # ##.
830	# Detect has_extglob so that DoUnarySuffixOp doesn't use the fast
831	# shortcut for constant strings.
832	arg_val, has_extglob = self.EvalWordToPattern(op.arg_word)
833	assert arg_val.tag() == value_e.Str
834
835	UP_val = val
836	with tagswitch(val) as case:
837	if case(value_e.Str):
838	val = cast(value.Str, UP_val)
839	s = string_ops.DoUnarySuffixOp(val.s, op.op, arg_val.s,
840	has_extglob)
841	#log('%r %r -> %r', val.s, arg_val.s, s)
842	new_val = value.Str(s) # type: value_t
843
844	elif case(value_e.BashArray):
845	val = cast(value.BashArray, UP_val)
846	# ${a[@]#prefix} is VECTORIZED on arrays. YSH should have this too.
847	strs = [] # type: List[str]
848	for s in val.strs:
849	if s is not None:
850	strs.append(
851	string_ops.DoUnarySuffixOp(
852	s, op.op, arg_val.s, has_extglob))
853	new_val = value.BashArray(strs)
854
855	elif case(value_e.BashAssoc):
856	val = cast(value.BashAssoc, UP_val)
857	strs = []
858	for s in val.d.values():
859	strs.append(
860	string_ops.DoUnarySuffixOp(s, op.op, arg_val.s,
861	has_extglob))
862	new_val = value.BashArray(strs)
863
864	else:
865	raise error.TypeErr(
866	val, 'Unary op expected Str, BashArray, BashAssoc',
867	op.op)
868
869	else:
870	raise AssertionError(Kind_str(op_kind))
871
872	return new_val
873
874	def _PatSub(self, val, op):
875	# type: (value_t, suffix_op.PatSub) -> value_t
876
877	pat_val, has_extglob = self.EvalWordToPattern(op.pat)
878	# Extended globs aren't supported because we only translate * ? etc. to
879	# ERE. I don't think there's a straightforward translation from !(*.py) to
880	# ERE! You would need an engine that supports negation? (Derivatives?)
881	if has_extglob:
882	e_die('extended globs not supported in ${x//GLOB/}', op.pat)
883
884	if op.replace:
885	replace_val = self.EvalRhsWord(op.replace)
886	# Can't have an array, so must be a string
887	assert replace_val.tag() == value_e.Str, replace_val
888	replace_str = cast(value.Str, replace_val).s
889	else:
890	replace_str = ''
891
892	# note: doesn't support self.exec_opts.extglob()!
893	regex, warnings = glob_.GlobToERE(pat_val.s)
894	if len(warnings):
895	# TODO:
896	# - Add 'shopt -s strict_glob' mode and expose warnings.
897	# "Glob is not in CANONICAL FORM".
898	# - Propagate location info back to the 'op.pat' word.
899	pass
900	replacer = string_ops.GlobReplacer(regex, replace_str, op.slash_tok)
901
902	with tagswitch(val) as case2:
903	if case2(value_e.Str):
904	str_val = cast(value.Str, val)
905	s = replacer.Replace(str_val.s, op)
906	val = value.Str(s)
907
908	elif case2(value_e.BashArray):
909	array_val = cast(value.BashArray, val)
910	strs = [] # type: List[str]
911	for s in array_val.strs:
912	if s is not None:
913	strs.append(replacer.Replace(s, op))
914	val = value.BashArray(strs)
915
916	elif case2(value_e.BashAssoc):
917	assoc_val = cast(value.BashAssoc, val)
918	strs = []
919	for s in assoc_val.d.values():
920	strs.append(replacer.Replace(s, op))
921	val = value.BashArray(strs)
922
923	else:
924	raise error.TypeErr(
925	val, 'Pat Sub op expected Str, BashArray, BashAssoc',
926	op.slash_tok)
927
928	return val
929
930	def _Slice(self, val, op, var_name, part):
931	# type: (value_t, suffix_op.Slice, Optional[str], BracedVarSub) -> value_t
932
933	if op.begin:
934	begin = self.arith_ev.EvalToInt(op.begin)
935	else:
936	begin = 0
937
938	# Note: bash allows lengths to be negative (with odd semantics), but
939	# we don't allow that right now.
940	has_length = False
941	length = -1
942	if op.length:
943	has_length = True
944	length = self.arith_ev.EvalToInt(op.length)
945
946	try:
947	arg0_val = None # type: value.Str
948	if var_name is None: # $* or $@
949	arg0_val = self.mem.GetArg0()
950	val = _PerformSlice(val, begin, length, has_length, part, arg0_val)
951	except error.Strict as e:
952	if self.exec_opts.strict_word_eval():
953	raise
954	else:
955	self.errfmt.PrettyPrintError(e, prefix='warning: ')
956	with tagswitch(val) as case2:
957	if case2(value_e.Str):
958	val = value.Str('')
959	elif case2(value_e.BashArray):
960	val = value.BashArray([])
961	else:
962	raise NotImplementedError()
963	return val
964
965	def _Nullary(self, val, op, var_name):
966	# type: (value_t, Token, Optional[str]) -> Tuple[value.Str, bool]
967
968	UP_val = val
969	quoted2 = False
970	op_id = op.id
971	if op_id == Id.VOp0_P:
972	with tagswitch(val) as case:
973	if case(value_e.Str):
974	str_val = cast(value.Str, UP_val)
975	prompt = self.prompt_ev.EvalPrompt(str_val)
976	# readline gets rid of these, so we should too.
977	p = prompt.replace('\x01', '').replace('\x02', '')
978	result = value.Str(p)
979	else:
980	e_die("Can't use @P on %s" % ui.ValType(val), op)
981
982	elif op_id == Id.VOp0_Q:
983	with tagswitch(val) as case:
984	if case(value_e.Str):
985	str_val = cast(value.Str, UP_val)
986
987	# TODO: use fastfunc.ShellEncode or
988	# fastfunc.PosixShellEncode()
989	result = value.Str(j8_lite.MaybeShellEncode(str_val.s))
990	# oddly, 'echo ${x@Q}' is equivalent to 'echo "${x@Q}"' in bash
991	quoted2 = True
992	elif case(value_e.BashArray):
993	array_val = cast(value.BashArray, UP_val)
994
995	# TODO: should use fastfunc.ShellEncode
996	tmp = [j8_lite.MaybeShellEncode(s) for s in array_val.strs]
997	result = value.Str(' '.join(tmp))
998	else:
999	e_die("Can't use @Q on %s" % ui.ValType(val), op)
1000
1001	elif op_id == Id.VOp0_a:
1002	# We're ONLY simluating -a and -A, not -r -x -n for now. See
1003	# spec/ble-idioms.test.sh.
1004	chars = [] # type: List[str]
1005	with tagswitch(val) as case:
1006	if case(value_e.BashArray):
1007	chars.append('a')
1008	elif case(value_e.BashAssoc):
1009	chars.append('A')
1010
1011	if var_name is not None: # e.g. ${?@a} is allowed
1012	cell = self.mem.GetCell(var_name)
1013	if cell:
1014	if cell.readonly:
1015	chars.append('r')
1016	if cell.exported:
1017	chars.append('x')
1018	if cell.nameref:
1019	chars.append('n')
1020
1021	result = value.Str(''.join(chars))
1022
1023	else:
1024	e_die('Var op %r not implemented' % lexer.TokenVal(op), op)
1025
1026	return result, quoted2
1027
1028	def _WholeArray(self, val, part, quoted, vsub_state):
1029	# type: (value_t, BracedVarSub, bool, VarSubState) -> value_t
1030	op_id = cast(bracket_op.WholeArray, part.bracket_op).op_id
1031
1032	if op_id == Id.Lit_At:
1033	vsub_state.join_array = not quoted # ${a[@]} decays but "${a[@]}" doesn't
1034	UP_val = val
1035	with tagswitch(val) as case2:
1036	if case2(value_e.Undef):
1037	if not vsub_state.has_test_op:
1038	val = self._EmptyBashArrayOrError(part.token)
1039	elif case2(value_e.Str):
1040	if self.exec_opts.strict_array():
1041	e_die("Can't index string with @", loc.WordPart(part))
1042	elif case2(value_e.BashArray):
1043	pass # no-op
1044
1045	elif op_id == Id.Arith_Star:
1046	vsub_state.join_array = True # both ${a[]} and "${a[]}" decay
1047	UP_val = val
1048	with tagswitch(val) as case2:
1049	if case2(value_e.Undef):
1050	if not vsub_state.has_test_op:
1051	val = self._EmptyBashArrayOrError(part.token)
1052	elif case2(value_e.Str):
1053	if self.exec_opts.strict_array():
1054	e_die("Can't index string with *", loc.WordPart(part))
1055	elif case2(value_e.BashArray):
1056	pass # no-op
1057
1058	else:
1059	raise AssertionError(op_id) # unknown
1060
1061	return val
1062
1063	def _ArrayIndex(self, val, part, vtest_place):
1064	# type: (value_t, BracedVarSub, VTestPlace) -> value_t
1065	"""Process a numeric array index like ${a[i+1]}"""
1066	anode = cast(bracket_op.ArrayIndex, part.bracket_op).expr
1067
1068	UP_val = val
1069	with tagswitch(val) as case2:
1070	if case2(value_e.Undef):
1071	pass # it will be checked later
1072
1073	elif case2(value_e.Str):
1074	# Bash treats any string as an array, so we can't add our own
1075	# behavior here without making valid OSH invalid bash.
1076	e_die("Can't index string %r with integer" % part.var_name,
1077	part.token)
1078
1079	elif case2(value_e.BashArray):
1080	array_val = cast(value.BashArray, UP_val)
1081	index = self.arith_ev.EvalToInt(anode)
1082	vtest_place.index = a_index.Int(index)
1083
1084	s = GetArrayItem(array_val.strs, index)
1085
1086	if s is None:
1087	val = value.Undef
1088	else:
1089	val = value.Str(s)
1090
1091	elif case2(value_e.BashAssoc):
1092	assoc_val = cast(value.BashAssoc, UP_val)
1093	# Location could also be attached to bracket_op? But
1094	# arith_expr.VarSub works OK too
1095	key = self.arith_ev.EvalWordToString(
1096	anode, blame_loc=location.TokenForArith(anode))
1097
1098	vtest_place.index = a_index.Str(key) # out param
1099	s = assoc_val.d.get(key)
1100
1101	if s is None:
1102	val = value.Undef
1103	else:
1104	val = value.Str(s)
1105
1106	else:
1107	raise error.TypeErr(val,
1108	'Index op expected BashArray, BashAssoc',
1109	loc.WordPart(part))
1110
1111	return val
1112
1113	def _EvalDoubleQuoted(self, parts, part_vals):
1114	# type: (List[word_part_t], List[part_value_t]) -> None
1115	"""Evaluate parts of a DoubleQuoted part.
1116
1117	Args:
1118	part_vals: output param to append to.
1119	"""
1120	# Example of returning array:
1121	# $ a=(1 2); b=(3); $ c=(4 5)
1122	# $ argv "${a[@]}${b[@]}${c[@]}"
1123	# ['1', '234', '5']
1124	#
1125	# Example of multiple parts
1126	# $ argv "${a[@]}${undef[@]:-${c[@]}}"
1127	# ['1', '24', '5']
1128
1129	# Special case for "". The parser outputs (DoubleQuoted []), instead
1130	# of (DoubleQuoted [Literal '']). This is better but it means we
1131	# have to check for it.
1132	if len(parts) == 0:
1133	v = Piece('', True, False)
1134	part_vals.append(v)
1135	return
1136
1137	for p in parts:
1138	self._EvalWordPart(p, part_vals, QUOTED)
1139
1140	def EvalDoubleQuotedToString(self, dq_part):
1141	# type: (DoubleQuoted) -> str
1142	"""For double quoted strings in YSH expressions.
1143
1144	Example: var x = "$foo-${foo}"
1145	"""
1146	part_vals = [] # type: List[part_value_t]
1147	self._EvalDoubleQuoted(dq_part.parts, part_vals)
1148	return self._ConcatPartVals(part_vals, dq_part.left)
1149
1150	def _DecayArray(self, val):
1151	# type: (value.BashArray) -> value.Str
1152	"""Decay $* to a string."""
1153	assert val.tag() == value_e.BashArray, val
1154	sep = self.splitter.GetJoinChar()
1155	tmp = [s for s in val.strs if s is not None]
1156	return value.Str(sep.join(tmp))
1157
1158	def _EmptyStrOrError(self, val, token):
1159	# type: (value_t, Token) -> value_t
1160	if val.tag() != value_e.Undef:
1161	return val
1162
1163	if not self.exec_opts.nounset():
1164	return value.Str('')
1165
1166	tok_str = lexer.TokenVal(token)
1167	name = tok_str[1:] if tok_str.startswith('$') else tok_str
1168	e_die('Undefined variable %r' % name, token)
1169
1170	def _EmptyBashArrayOrError(self, token):
1171	# type: (Token) -> value_t
1172	assert token is not None
1173	if self.exec_opts.nounset():
1174	e_die('Undefined array %r' % lexer.TokenVal(token), token)
1175	else:
1176	return value.BashArray([])
1177
1178	def _EvalBracketOp(self, val, part, quoted, vsub_state, vtest_place):
1179	# type: (value_t, BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1180
1181	if part.bracket_op:
1182	with tagswitch(part.bracket_op) as case:
1183	if case(bracket_op_e.WholeArray):
1184	val = self._WholeArray(val, part, quoted, vsub_state)
1185
1186	elif case(bracket_op_e.ArrayIndex):
1187	val = self._ArrayIndex(val, part, vtest_place)
1188
1189	else:
1190	raise AssertionError(part.bracket_op.tag())
1191
1192	else: # no bracket op
1193	var_name = vtest_place.name
1194	if (var_name is not None and
1195	val.tag() in (value_e.BashArray, value_e.BashAssoc) and
1196	not vsub_state.is_type_query):
1197	if ShouldArrayDecay(var_name, self.exec_opts,
1198	not (part.prefix_op or part.suffix_op)):
1199	# for ${BASH_SOURCE}, etc.
1200	val = DecayArray(val)
1201	else:
1202	e_die(
1203	"Array %r can't be referred to as a scalar (without @ or *)"
1204	% var_name, loc.WordPart(part))
1205
1206	return val
1207
1208	def _VarRefValue(self, part, quoted, vsub_state, vtest_place):
1209	# type: (BracedVarSub, bool, VarSubState, VTestPlace) -> value_t
1210	"""Duplicates some logic from _EvalBracedVarSub, but returns a
1211	value_t."""
1212
1213	# 1. Evaluate from (var_name, var_num, token Id) -> value
1214	if part.token.id == Id.VSub_Name:
1215	vtest_place.name = part.var_name
1216	val = self.mem.GetValue(part.var_name)
1217
1218	elif part.token.id == Id.VSub_Number:
1219	var_num = int(part.var_name)
1220	val = self._EvalVarNum(var_num)
1221
1222	else:
1223	# $* decays
1224	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1225
1226	# We don't need var_index because it's only for L-Values of test ops?
1227	if self.exec_opts.eval_unsafe_arith():
1228	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1229	vtest_place)
1230	else:
1231	with state.ctx_Option(self.mutable_opts,
1232	[option_i._allow_command_sub], False):
1233	val = self._EvalBracketOp(val, part, quoted, vsub_state,
1234	vtest_place)
1235
1236	return val
1237
1238	def _EvalBracedVarSub(self, part, part_vals, quoted):
1239	# type: (BracedVarSub, List[part_value_t], bool) -> None
1240	"""
1241	Args:
1242	part_vals: output param to append to.
1243	"""
1244	# We have different operators that interact in a non-obvious order.
1245	#
1246	# 1. bracket_op: value -> value, with side effect on vsub_state
1247	#
1248	# 2. prefix_op
1249	# a. length ${#x}: value -> value
1250	# b. var ref ${!ref}: can expand to an array
1251	#
1252	# 3. suffix_op:
1253	# a. no operator: you have a value
1254	# b. Test: value -> part_value[]
1255	# c. Other Suffix: value -> value
1256	#
1257	# 4. Process vsub_state.join_array here before returning.
1258	#
1259	# These cases are hard to distinguish:
1260	# - ${!prefix@} prefix query
1261	# - ${!array[@]} keys
1262	# - ${!ref} named reference
1263	# - ${!ref[0]} named reference
1264	#
1265	# I think we need several stages:
1266	#
1267	# 1. value: name, number, special, prefix query
1268	# 2. bracket_op
1269	# 3. prefix length -- this is TERMINAL
1270	# 4. indirection? Only for some of the ! cases
1271	# 5. string transformation suffix ops like ##
1272	# 6. test op
1273	# 7. vsub_state.join_array
1274
1275	# vsub_state.join_array is for joining "${a[*]}" and unquoted ${a[@]} AFTER
1276	# suffix ops are applied. If we take the length with a prefix op, the
1277	# distinction is ignored.
1278
1279	var_name = None # type: Optional[str] # used throughout the function
1280	vtest_place = VTestPlace(var_name, None) # For ${foo=default}
1281	vsub_state = VarSubState.CreateNull() # for $, ${a[]}, etc.
1282
1283	# 1. Evaluate from (var_name, var_num, token Id) -> value
1284	if part.token.id == Id.VSub_Name:
1285	# Handle ${!prefix@} first, since that looks at names and not values
1286	# Do NOT handle ${!A[@]@a} here!
1287	if (part.prefix_op is not None and part.bracket_op is None and
1288	part.suffix_op is not None and
1289	part.suffix_op.tag() == suffix_op_e.Nullary):
1290	nullary_op = cast(Token, part.suffix_op)
1291	# ${!x@} but not ${!x@P}
1292	if consts.GetKind(nullary_op.id) == Kind.VOp3:
1293	names = self.mem.VarNamesStartingWith(part.var_name)
1294	names.sort()
1295
1296	if quoted and nullary_op.id == Id.VOp3_At:
1297	part_vals.append(part_value.Array(names))
1298	else:
1299	sep = self.splitter.GetJoinChar()
1300	part_vals.append(Piece(sep.join(names), quoted, True))
1301	return # EARLY RETURN
1302
1303	var_name = part.var_name
1304	vtest_place.name = var_name # for _ApplyTestOp
1305
1306	val = self.mem.GetValue(var_name)
1307
1308	elif part.token.id == Id.VSub_Number:
1309	var_num = int(part.var_name)
1310	val = self._EvalVarNum(var_num)
1311	else:
1312	# $* decays
1313	val = self._EvalSpecialVar(part.token.id, quoted, vsub_state)
1314
1315	suffix_op_ = part.suffix_op
1316	if suffix_op_:
1317	UP_op = suffix_op_
1318	with tagswitch(suffix_op_) as case:
1319	if case(suffix_op_e.Nullary):
1320	suffix_op_ = cast(Token, UP_op)
1321
1322	# Type query ${array@a} is a STRING, not an array
1323	# NOTE: ${array@Q} is ${array[0]@Q} in bash, which is different than
1324	# ${array[@]@Q}
1325	if suffix_op_.id == Id.VOp0_a:
1326	vsub_state.is_type_query = True
1327
1328	elif case(suffix_op_e.Unary):
1329	suffix_op_ = cast(suffix_op.Unary, UP_op)
1330
1331	# Do the _EmptyStrOrError/_EmptyBashArrayOrError up front, EXCEPT in
1332	# the case of Kind.VTest
1333	if consts.GetKind(suffix_op_.op.id) == Kind.VTest:
1334	vsub_state.has_test_op = True
1335
1336	# 2. Bracket Op
1337	val = self._EvalBracketOp(val, part, quoted, vsub_state, vtest_place)
1338
1339	if part.prefix_op:
1340	if part.prefix_op.id == Id.VSub_Pound: # ${#var} for length
1341	if not vsub_state.has_test_op: # undef -> '' BEFORE length
1342	val = self._EmptyStrOrError(val, part.token)
1343
1344	n = self._Length(val, part.token)
1345	part_vals.append(Piece(str(n), quoted, False))
1346	return # EARLY EXIT: nothing else can come after length
1347
1348	elif part.prefix_op.id == Id.VSub_Bang:
1349	if (part.bracket_op and
1350	part.bracket_op.tag() == bracket_op_e.WholeArray):
1351	if vsub_state.has_test_op:
1352	# ${!a[@]-'default'} is a non-fatal runtime error in bash. Here
1353	# it's fatal.
1354	op_tok = cast(suffix_op.Unary, UP_op).op
1355	e_die('Test operation not allowed with ${!array[@]}',
1356	op_tok)
1357
1358	# ${!array[@]} to get indices/keys
1359	val = self._Keys(val, part.token)
1360	# already set vsub_State.join_array ABOVE
1361	else:
1362	# Process ${!ref}. SURPRISE: ${!a[0]} is an indirect expansion unlike
1363	# ${!a[@]} !
1364	# ${!ref} can expand into an array if ref='array[@]'
1365
1366	# Clear it now that we have a var ref
1367	vtest_place.name = None
1368	vtest_place.index = None
1369
1370	val = self._EvalVarRef(val, part.token, quoted, vsub_state,
1371	vtest_place)
1372
1373	if not vsub_state.has_test_op: # undef -> '' AFTER indirection
1374	val = self._EmptyStrOrError(val, part.token)
1375
1376	else:
1377	raise AssertionError(part.prefix_op)
1378
1379	else:
1380	if not vsub_state.has_test_op: # undef -> '' if no prefix op
1381	val = self._EmptyStrOrError(val, part.token)
1382
1383	quoted2 = False # another bit for @Q
1384	if suffix_op_:
1385	op = suffix_op_ # could get rid of this alias
1386
1387	with tagswitch(suffix_op_) as case:
1388	if case(suffix_op_e.Nullary):
1389	op = cast(Token, UP_op)
1390	val, quoted2 = self._Nullary(val, op, var_name)
1391
1392	elif case(suffix_op_e.Unary):
1393	op = cast(suffix_op.Unary, UP_op)
1394	if consts.GetKind(op.op.id) == Kind.VTest:
1395	if self._ApplyTestOp(val, op, quoted, part_vals,
1396	vtest_place, part.token):
1397	# e.g. to evaluate ${undef:-'default'}, we already appended
1398	# what we need
1399	return
1400
1401	else:
1402	# Other suffix: value -> value
1403	val = self._ApplyUnarySuffixOp(val, op)
1404
1405	elif case(suffix_op_e.PatSub): # PatSub, vectorized
1406	op = cast(suffix_op.PatSub, UP_op)
1407	val = self._PatSub(val, op)
1408
1409	elif case(suffix_op_e.Slice):
1410	op = cast(suffix_op.Slice, UP_op)
1411	val = self._Slice(val, op, var_name, part)
1412
1413	elif case(suffix_op_e.Static):
1414	op = cast(suffix_op.Static, UP_op)
1415	e_die('Not implemented', op.tok)
1416
1417	else:
1418	raise AssertionError()
1419
1420	# After applying suffixes, process join_array here.
1421	UP_val = val
1422	if val.tag() == value_e.BashArray:
1423	array_val = cast(value.BashArray, UP_val)
1424	if vsub_state.join_array:
1425	val = self._DecayArray(array_val)
1426	else:
1427	val = array_val
1428
1429	# For example, ${a} evaluates to value.Str(), but we want a
1430	# Piece().
1431	part_val = _ValueToPartValue(val, quoted or quoted2, part)
1432	part_vals.append(part_val)
1433
1434	def _ConcatPartVals(self, part_vals, location):
1435	# type: (List[part_value_t], loc_t) -> str
1436
1437	strs = [] # type: List[str]
1438	for part_val in part_vals:
1439	UP_part_val = part_val
1440	with tagswitch(part_val) as case:
1441	if case(part_value_e.String):
1442	part_val = cast(Piece, UP_part_val)
1443	s = part_val.s
1444
1445	elif case(part_value_e.Array):
1446	part_val = cast(part_value.Array, UP_part_val)
1447	if self.exec_opts.strict_array():
1448	# Examples: echo f > "$@"; local foo="$@"
1449	e_die("Illegal array word part (strict_array)",
1450	location)
1451	else:
1452	# It appears to not respect IFS
1453	# TODO: eliminate double join()?
1454	tmp = [s for s in part_val.strs if s is not None]
1455	s = ' '.join(tmp)
1456
1457	else:
1458	raise AssertionError()
1459
1460	strs.append(s)
1461
1462	return ''.join(strs)
1463
1464	def EvalBracedVarSubToString(self, part):
1465	# type: (BracedVarSub) -> str
1466	"""For double quoted strings in YSH expressions.
1467
1468	Example: var x = "$foo-${foo}"
1469	"""
1470	part_vals = [] # type: List[part_value_t]
1471	self._EvalBracedVarSub(part, part_vals, False)
1472	# blame ${ location
1473	return self._ConcatPartVals(part_vals, part.left)
1474
1475	def _EvalSimpleVarSub(self, part, part_vals, quoted):
1476	# type: (SimpleVarSub, List[part_value_t], bool) -> None
1477
1478	token = part.tok
1479
1480	vsub_state = VarSubState.CreateNull()
1481
1482	# 1. Evaluate from (var_name, var_num, Token) -> defined, value
1483	if token.id == Id.VSub_DollarName:
1484	var_name = lexer.LazyStr(token)
1485	# TODO: Special case for LINENO
1486	val = self.mem.GetValue(var_name)
1487	if val.tag() in (value_e.BashArray, value_e.BashAssoc):
1488	if ShouldArrayDecay(var_name, self.exec_opts):
1489	# for $BASH_SOURCE, etc.
1490	val = DecayArray(val)
1491	else:
1492	e_die(
1493	"Array %r can't be referred to as a scalar (without @ or *)"
1494	% var_name, token)
1495
1496	elif token.id == Id.VSub_Number:
1497	var_num = int(lexer.LazyStr(token))
1498	val = self._EvalVarNum(var_num)
1499
1500	else:
1501	val = self._EvalSpecialVar(token.id, quoted, vsub_state)
1502
1503	#log('SIMPLE %s', part)
1504	val = self._EmptyStrOrError(val, token)
1505	UP_val = val
1506	if val.tag() == value_e.BashArray:
1507	array_val = cast(value.BashArray, UP_val)
1508	if vsub_state.join_array:
1509	val = self._DecayArray(array_val)
1510	else:
1511	val = array_val
1512
1513	v = _ValueToPartValue(val, quoted, part)
1514	part_vals.append(v)
1515
1516	def EvalSimpleVarSubToString(self, node):
1517	# type: (SimpleVarSub) -> str
1518	"""For double quoted strings in YSH expressions.
1519
1520	Example: var x = "$foo-${foo}"
1521	"""
1522	part_vals = [] # type: List[part_value_t]
1523	self._EvalSimpleVarSub(node, part_vals, False)
1524	return self._ConcatPartVals(part_vals, node.tok)
1525
1526	def _EvalExtGlob(self, part, part_vals):
1527	# type: (word_part.ExtGlob, List[part_value_t]) -> None
1528	"""Evaluate @($x\|'foo'\|$(hostname)) and flatten it."""
1529	op = part.op
1530	if op.id == Id.ExtGlob_Comma:
1531	op_str = '@('
1532	else:
1533	op_str = lexer.LazyStr(op)
1534	# Do NOT split these.
1535	part_vals.append(Piece(op_str, False, False))
1536
1537	for i, w in enumerate(part.arms):
1538	if i != 0:
1539	part_vals.append(Piece('\|', False, False)) # separator
1540	# FLATTEN the tree of extglob "arms".
1541	self._EvalWordToParts(w, part_vals, EXTGLOB_NESTED)
1542	part_vals.append(Piece(')', False, False)) # closing )
1543
1544	def _TranslateExtGlob(self, part_vals, w, glob_parts, fnmatch_parts):
1545	# type: (List[part_value_t], CompoundWord, List[str], List[str]) -> None
1546	"""Translate a flattened WORD with an ExtGlob part to string patterns.
1547
1548	We need both glob and fnmatch patterns. _EvalExtGlob does the
1549	flattening.
1550	"""
1551	for i, part_val in enumerate(part_vals):
1552	UP_part_val = part_val
1553	with tagswitch(part_val) as case:
1554	if case(part_value_e.String):
1555	part_val = cast(Piece, UP_part_val)
1556	if part_val.quoted and not self.exec_opts.noglob():
1557	s = glob_.GlobEscape(part_val.s)
1558	else:
1559	# e.g. the @( and \| in @(foo\|bar) aren't quoted
1560	s = part_val.s
1561	glob_parts.append(s)
1562	fnmatch_parts.append(s) # from _EvalExtGlob()
1563
1564	elif case(part_value_e.Array):
1565	# Disallow array
1566	e_die(
1567	"Extended globs and arrays can't appear in the same word",
1568	w)
1569
1570	elif case(part_value_e.ExtGlob):
1571	part_val = cast(part_value.ExtGlob, UP_part_val)
1572	# keep appending fnmatch_parts, but repplace glob_parts with '*'
1573	self._TranslateExtGlob(part_val.part_vals, w, [],
1574	fnmatch_parts)
1575	glob_parts.append('*')
1576
1577	else:
1578	raise AssertionError()
1579
1580	def _EvalWordPart(self, part, part_vals, flags):
1581	# type: (word_part_t, List[part_value_t], int) -> None
1582	"""Evaluate a word part, appending to part_vals
1583
1584	Called by _EvalWordToParts, EvalWordToString, and _EvalDoubleQuoted.
1585	"""
1586	quoted = bool(flags & QUOTED)
1587	is_subst = bool(flags & IS_SUBST)
1588
1589	UP_part = part
1590	with tagswitch(part) as case:
1591	if case(word_part_e.ShArrayLiteral):
1592	part = cast(ShArrayLiteral, UP_part)
1593	e_die("Unexpected array literal", loc.WordPart(part))
1594	elif case(word_part_e.BashAssocLiteral):
1595	part = cast(word_part.BashAssocLiteral, UP_part)
1596	e_die("Unexpected associative array literal",
1597	loc.WordPart(part))
1598
1599	elif case(word_part_e.Literal):
1600	part = cast(Token, UP_part)
1601	# Split if it's in a substitution.
1602	# That is: echo is not split, but ${foo:-echo} is split
1603	v = Piece(lexer.LazyStr(part), quoted, is_subst)
1604	part_vals.append(v)
1605
1606	elif case(word_part_e.EscapedLiteral):
1607	part = cast(word_part.EscapedLiteral, UP_part)
1608	v = Piece(part.ch, True, False)
1609	part_vals.append(v)
1610
1611	elif case(word_part_e.SingleQuoted):
1612	part = cast(SingleQuoted, UP_part)
1613	v = Piece(part.sval, True, False)
1614	part_vals.append(v)
1615
1616	elif case(word_part_e.DoubleQuoted):
1617	part = cast(DoubleQuoted, UP_part)
1618	self._EvalDoubleQuoted(part.parts, part_vals)
1619
1620	elif case(word_part_e.CommandSub):
1621	part = cast(CommandSub, UP_part)
1622	id_ = part.left_token.id
1623	if id_ in (Id.Left_DollarParen, Id.Left_AtParen,
1624	Id.Left_Backtick):
1625	sv = self._EvalCommandSub(part,
1626	quoted) # type: part_value_t
1627
1628	elif id_ in (Id.Left_ProcSubIn, Id.Left_ProcSubOut):
1629	sv = self._EvalProcessSub(part)
1630
1631	else:
1632	raise AssertionError(id_)
1633
1634	part_vals.append(sv)
1635
1636	elif case(word_part_e.SimpleVarSub):
1637	part = cast(SimpleVarSub, UP_part)
1638	self._EvalSimpleVarSub(part, part_vals, quoted)
1639
1640	elif case(word_part_e.BracedVarSub):
1641	part = cast(BracedVarSub, UP_part)
1642	self._EvalBracedVarSub(part, part_vals, quoted)
1643
1644	elif case(word_part_e.TildeSub):
1645	part = cast(word_part.TildeSub, UP_part)
1646	# We never parse a quoted string into a TildeSub.
1647	assert not quoted
1648	s = self.tilde_ev.Eval(part)
1649	v = Piece(s, True, False) # NOT split even when unquoted!
1650	part_vals.append(v)
1651
1652	elif case(word_part_e.ArithSub):
1653	part = cast(word_part.ArithSub, UP_part)
1654	num = self.arith_ev.EvalToBigInt(part.anode)
1655	v = Piece(mops.ToStr(num), quoted, not quoted)
1656	part_vals.append(v)
1657
1658	elif case(word_part_e.ExtGlob):
1659	part = cast(word_part.ExtGlob, UP_part)
1660	#if not self.exec_opts.extglob():
1661	# die() # disallow at runtime? Don't just decay
1662
1663	# Create a node to hold the flattened tree. The caller decides whether
1664	# to pass it to fnmatch() or replace it with '*' and pass it to glob().
1665	part_vals2 = [] # type: List[part_value_t]
1666	self._EvalExtGlob(part, part_vals2) # flattens tree
1667	part_vals.append(part_value.ExtGlob(part_vals2))
1668
1669	elif case(word_part_e.BashRegexGroup):
1670	part = cast(word_part.BashRegexGroup, UP_part)
1671
1672	part_vals.append(Piece('(', False, False)) # not quoted
1673	if part.child:
1674	self._EvalWordToParts(part.child, part_vals, 0)
1675	part_vals.append(Piece(')', False, False))
1676
1677	elif case(word_part_e.Splice):
1678	part = cast(word_part.Splice, UP_part)
1679	val = self.mem.GetValue(part.var_name)
1680
1681	strs = self.expr_ev.SpliceValue(val, part)
1682	part_vals.append(part_value.Array(strs))
1683
1684	elif case(word_part_e.ExprSub):
1685	part = cast(word_part.ExprSub, UP_part)
1686	part_val = self.expr_ev.EvalExprSub(part)
1687	part_vals.append(part_val)
1688
1689	elif case(word_part_e.ZshVarSub):
1690	part = cast(word_part.ZshVarSub, UP_part)
1691	e_die("ZSH var subs are parsed, but can't be evaluated",
1692	part.left)
1693
1694	else:
1695	raise AssertionError(part.tag())
1696
1697	def _EvalRhsWordToParts(self, w, part_vals, eval_flags=0):
1698	# type: (rhs_word_t, List[part_value_t], int) -> None
1699	quoted = bool(eval_flags & QUOTED)
1700
1701	UP_w = w
1702	with tagswitch(w) as case:
1703	if case(rhs_word_e.Empty):
1704	part_vals.append(Piece('', quoted, not quoted))
1705
1706	elif case(rhs_word_e.Compound):
1707	w = cast(CompoundWord, UP_w)
1708	self._EvalWordToParts(w, part_vals, eval_flags=eval_flags)
1709
1710	else:
1711	raise AssertionError()
1712
1713	def _EvalWordToParts(self, w, part_vals, eval_flags=0):
1714	# type: (CompoundWord, List[part_value_t], int) -> None
1715	"""Helper for EvalRhsWord, EvalWordSequence, etc.
1716
1717	Returns:
1718	Appends to part_vals. Note that this is a TREE.
1719	"""
1720	# Does the word have an extended glob? This is a special case because
1721	# of the way we use glob() and then fnmatch(..., FNM_EXTMATCH) to
1722	# implement extended globs. It's hard to carry that extra information
1723	# all the way past the word splitting stage.
1724
1725	# OSH semantic limitations: If a word has an extended glob part, then
1726	# 1. It can't have an array
1727	# 2. Word splitting of unquoted words isn't respected
1728
1729	word_part_vals = [] # type: List[part_value_t]
1730	has_extglob = False
1731	for p in w.parts:
1732	if p.tag() == word_part_e.ExtGlob:
1733	has_extglob = True
1734	self._EvalWordPart(p, word_part_vals, eval_flags)
1735
1736	# Caller REQUESTED extglob evaluation, AND we parsed word_part.ExtGlob()
1737	if has_extglob:
1738	if bool(eval_flags & EXTGLOB_FILES):
1739	# Treat the WHOLE word as a pattern. We need to TWO VARIANTS of the
1740	# word because of the way we use libc:
1741	# 1. With '*' for extglob parts
1742	# 2. With _EvalExtGlob() for extglob parts
1743
1744	glob_parts = [] # type: List[str]
1745	fnmatch_parts = [] # type: List[str]
1746	self._TranslateExtGlob(word_part_vals, w, glob_parts,
1747	fnmatch_parts)
1748
1749	#log('word_part_vals %s', word_part_vals)
1750	glob_pat = ''.join(glob_parts)
1751	fnmatch_pat = ''.join(fnmatch_parts)
1752	#log("glob %s fnmatch %s", glob_pat, fnmatch_pat)
1753
1754	results = [] # type: List[str]
1755	n = self.globber.ExpandExtended(glob_pat, fnmatch_pat, results)
1756	if n < 0:
1757	raise error.FailGlob(
1758	'Extended glob %r matched no files' % fnmatch_pat, w)
1759
1760	part_vals.append(part_value.Array(results))
1761	elif bool(eval_flags & EXTGLOB_NESTED):
1762	# We only glob at the TOP level of @(nested\|@(pattern))
1763	part_vals.extend(word_part_vals)
1764	else:
1765	# e.g. simple_word_eval, assignment builtin
1766	e_die('Extended glob not allowed in this word', w)
1767	else:
1768	part_vals.extend(word_part_vals)
1769
1770	def _PartValsToString(self, part_vals, w, eval_flags, strs):
1771	# type: (List[part_value_t], CompoundWord, int, List[str]) -> None
1772	"""Helper for EvalWordToString, similar to _ConcatPartVals() above.
1773
1774	Note: arg 'w' could just be a span ID
1775	"""
1776	for part_val in part_vals:
1777	UP_part_val = part_val
1778	with tagswitch(part_val) as case:
1779	if case(part_value_e.String):
1780	part_val = cast(Piece, UP_part_val)
1781	s = part_val.s
1782	if part_val.quoted:
1783	if eval_flags & QUOTE_FNMATCH:
1784	# [[ foo == /"".py ]] or case (.py) or ${x%.py} or ${x//*.py/}
1785	s = glob_.GlobEscape(s)
1786	elif eval_flags & QUOTE_ERE:
1787	s = glob_.ExtendedRegexEscape(s)
1788	strs.append(s)
1789
1790	elif case(part_value_e.Array):
1791	part_val = cast(part_value.Array, UP_part_val)
1792	if self.exec_opts.strict_array():
1793	# Examples: echo f > "$@"; local foo="$@"
1794
1795	# TODO: This attributes too coarsely, to the word rather than the
1796	# parts. Problem: the word is a TREE of parts, but we only have a
1797	# flat list of part_vals. The only case where we really get arrays
1798	# is "$@", "${a[@]}", "${a[@]//pat/replace}", etc.
1799	e_die(
1800	"This word should yield a string, but it contains an array",
1801	w)
1802
1803	# TODO: Maybe add detail like this.
1804	#e_die('RHS of assignment should only have strings. '
1805	# 'To assign arrays, use b=( "${a[@]}" )')
1806	else:
1807	# It appears to not respect IFS
1808	tmp = [s for s in part_val.strs if s is not None]
1809	s = ' '.join(tmp) # TODO: eliminate double join()?
1810	strs.append(s)
1811
1812	elif case(part_value_e.ExtGlob):
1813	part_val = cast(part_value.ExtGlob, UP_part_val)
1814
1815	# Extended globs are only allowed where we expect them!
1816	if not bool(eval_flags & QUOTE_FNMATCH):
1817	e_die('extended glob not allowed in this word', w)
1818
1819	# recursive call
1820	self._PartValsToString(part_val.part_vals, w, eval_flags,
1821	strs)
1822
1823	else:
1824	raise AssertionError()
1825
1826	def EvalWordToString(self, UP_w, eval_flags=0):
1827	# type: (word_t, int) -> value.Str
1828	"""Given a word, return a string.
1829
1830	Flags can contain a quoting algorithm.
1831	"""
1832	assert UP_w.tag() == word_e.Compound, UP_w
1833	w = cast(CompoundWord, UP_w)
1834
1835	if eval_flags == 0: # QUOTE_FNMATCH etc. breaks optimization
1836	fast_str = word_.FastStrEval(w)
1837	if fast_str is not None:
1838	return value.Str(fast_str)
1839
1840	# Could we additionally optimize a=$b, if we know $b isn't an array
1841	# etc.?
1842
1843	# Note: these empty lists are hot in fib benchmark
1844
1845	part_vals = [] # type: List[part_value_t]
1846	for p in w.parts:
1847	# this doesn't use eval_flags, which is slightly confusing
1848	self._EvalWordPart(p, part_vals, 0)
1849
1850	strs = [] # type: List[str]
1851	self._PartValsToString(part_vals, w, eval_flags, strs)
1852	return value.Str(''.join(strs))
1853
1854	def EvalWordToPattern(self, UP_w):
1855	# type: (rhs_word_t) -> Tuple[value.Str, bool]
1856	"""Like EvalWordToString, but returns whether we got ExtGlob."""
1857	if UP_w.tag() == rhs_word_e.Empty:
1858	return value.Str(''), False
1859
1860	assert UP_w.tag() == rhs_word_e.Compound, UP_w
1861	w = cast(CompoundWord, UP_w)
1862
1863	has_extglob = False
1864	part_vals = [] # type: List[part_value_t]
1865	for p in w.parts:
1866	# this doesn't use eval_flags, which is slightly confusing
1867	self._EvalWordPart(p, part_vals, 0)
1868	if p.tag() == word_part_e.ExtGlob:
1869	has_extglob = True
1870
1871	strs = [] # type: List[str]
1872	self._PartValsToString(part_vals, w, QUOTE_FNMATCH, strs)
1873	return value.Str(''.join(strs)), has_extglob
1874
1875	def EvalForPlugin(self, w):
1876	# type: (CompoundWord) -> value.Str
1877	"""Wrapper around EvalWordToString that prevents errors.
1878
1879	Runtime errors like $(( 1 / 0 )) and mutating $? like $(exit 42)
1880	are handled here.
1881
1882	Similar to ExprEvaluator.PluginCall().
1883	"""
1884	with state.ctx_Registers(self.mem): # to "sandbox" $? and $PIPESTATUS
1885	try:
1886	val = self.EvalWordToString(w)
1887	except error.FatalRuntime as e:
1888	val = value.Str('<Runtime error: %s>' % e.UserErrorString())
1889
1890	except (IOError, OSError) as e:
1891	val = value.Str('<I/O error: %s>' % pyutil.strerror(e))
1892
1893	except KeyboardInterrupt:
1894	val = value.Str('<Ctrl-C>')
1895
1896	return val
1897
1898	def EvalRhsWord(self, UP_w):
1899	# type: (rhs_word_t) -> value_t
1900	"""Used for RHS of assignment.
1901
1902	There is no splitting.
1903	"""
1904	if UP_w.tag() == rhs_word_e.Empty:
1905	return value.Str('')
1906
1907	assert UP_w.tag() == word_e.Compound, UP_w
1908	w = cast(CompoundWord, UP_w)
1909
1910	if len(w.parts) == 1:
1911	part0 = w.parts[0]
1912	UP_part0 = part0
1913	tag = part0.tag()
1914	# Special case for a=(1 2). ShArrayLiteral won't appear in words that
1915	# don't look like assignments.
1916	if tag == word_part_e.ShArrayLiteral:
1917	part0 = cast(ShArrayLiteral, UP_part0)
1918	array_words = part0.words
1919	words = braces.BraceExpandWords(array_words)
1920	strs = self.EvalWordSequence(words)
1921	return value.BashArray(strs)
1922
1923	if tag == word_part_e.BashAssocLiteral:
1924	part0 = cast(word_part.BashAssocLiteral, UP_part0)
1925	d = NewDict() # type: Dict[str, str]
1926	for pair in part0.pairs:
1927	k = self.EvalWordToString(pair.key)
1928	v = self.EvalWordToString(pair.value)
1929	d[k.s] = v.s
1930	return value.BashAssoc(d)
1931
1932	# If RHS doesn't look like a=( ... ), then it must be a string.
1933	return self.EvalWordToString(w)
1934
1935	def _EvalWordFrame(self, frame, argv):
1936	# type: (List[Piece], List[str]) -> None
1937	all_empty = True
1938	all_quoted = True
1939	any_quoted = False
1940
1941	#log('--- frame %s', frame)
1942
1943	for piece in frame:
1944	if len(piece.s):
1945	all_empty = False
1946
1947	if piece.quoted:
1948	any_quoted = True
1949	else:
1950	all_quoted = False
1951
1952	# Elision of ${empty}${empty} but not $empty"$empty" or $empty""
1953	if all_empty and not any_quoted:
1954	return
1955
1956	# If every frag is quoted, e.g. "$a$b" or any part in "${a[@]}"x, then
1957	# don't do word splitting or globbing.
1958	if all_quoted:
1959	tmp = [piece.s for piece in frame]
1960	a = ''.join(tmp)
1961	argv.append(a)
1962	return
1963
1964	will_glob = not self.exec_opts.noglob()
1965
1966	# Array of strings, some of which are BOTH IFS-escaped and GLOB escaped!
1967	frags = [] # type: List[str]
1968	for piece in frame:
1969	if will_glob and piece.quoted:
1970	frag = glob_.GlobEscape(piece.s)
1971	else:
1972	# If we have a literal \, then we turn it into \\\\.
1973	# Splitting takes \\\\ -> \\
1974	# Globbing takes \\ to \ if it doesn't match
1975	frag = _BackslashEscape(piece.s)
1976
1977	if piece.do_split:
1978	frag = _BackslashEscape(frag)
1979	else:
1980	frag = self.splitter.Escape(frag)
1981
1982	frags.append(frag)
1983
1984	flat = ''.join(frags)
1985	#log('flat: %r', flat)
1986
1987	args = self.splitter.SplitForWordEval(flat)
1988
1989	# space=' '; argv $space"". We have a quoted part, but we CANNOT elide.
1990	# Add it back and don't bother globbing.
1991	if len(args) == 0 and any_quoted:
1992	argv.append('')
1993	return
1994
1995	#log('split args: %r', args)
1996	for a in args:
1997	if glob_.LooksLikeGlob(a):
1998	n = self.globber.Expand(a, argv)
1999	if n < 0:
2000	# TODO: location info, with span IDs carried through the frame
2001	raise error.FailGlob('Pattern %r matched no files' % a,
2002	loc.Missing)
2003	else:
2004	argv.append(glob_.GlobUnescape(a))
2005
2006	def _EvalWordToArgv(self, w):
2007	# type: (CompoundWord) -> List[str]
2008	"""Helper for _EvalAssignBuiltin.
2009
2010	Splitting and globbing are disabled for assignment builtins.
2011
2012	Example: declare -"${a[@]}" b=(1 2)
2013	where a is [x b=a d=a]
2014	"""
2015	part_vals = [] # type: List[part_value_t]
2016	self._EvalWordToParts(w, part_vals, 0) # not double quoted
2017	frames = _MakeWordFrames(part_vals)
2018	argv = [] # type: List[str]
2019	for frame in frames:
2020	if len(frame): # empty array gives empty frame!
2021	tmp = [piece.s for piece in frame]
2022	argv.append(''.join(tmp)) # no split or glob
2023	#log('argv: %s', argv)
2024	return argv
2025
2026	def _EvalAssignBuiltin(self, builtin_id, arg0, words):
2027	# type: (builtin_t, str, List[CompoundWord]) -> cmd_value.Assign
2028	"""Handles both static and dynamic assignment, e.g.
2029
2030	x='foo=bar' local a=(1 2) $x
2031	"""
2032	# Grammar:
2033	#
2034	# ('builtin' \| 'command')* keyword flag* pair*
2035	# flag = [-+].*
2036	#
2037	# There is also command -p, but we haven't implemented it. Maybe just punt
2038	# on it. Punted on 'builtin' and 'command' for now too.
2039
2040	eval_to_pairs = True # except for -f and -F
2041	started_pairs = False
2042
2043	flags = [arg0] # initial flags like -p, and -f -F name1 name2
2044	flag_locs = [words[0]]
2045	assign_args = [] # type: List[AssignArg]
2046
2047	n = len(words)
2048	for i in xrange(1, n): # skip first word
2049	w = words[i]
2050
2051	if word_.IsVarLike(w):
2052	started_pairs = True # Everything from now on is an assign_pair
2053
2054	if started_pairs:
2055	left_token, close_token, part_offset = word_.DetectShAssignment(
2056	w)
2057	if left_token: # Detected statically
2058	if left_token.id != Id.Lit_VarLike:
2059	# (not guaranteed since started_pairs is set twice)
2060	e_die('LHS array not allowed in assignment builtin', w)
2061
2062	if lexer.IsPlusEquals(left_token):
2063	var_name = lexer.TokenSliceRight(left_token, -2)
2064	append = True
2065	else:
2066	var_name = lexer.TokenSliceRight(left_token, -1)
2067	append = False
2068
2069	if part_offset == len(w.parts):
2070	rhs = rhs_word.Empty # type: rhs_word_t
2071	else:
2072	# tmp is for intersection of C++/MyPy type systems
2073	tmp = CompoundWord(w.parts[part_offset:])
2074	word_.TildeDetectAssign(tmp)
2075	rhs = tmp
2076
2077	with state.ctx_AssignBuiltin(self.mutable_opts):
2078	right = self.EvalRhsWord(rhs)
2079
2080	arg2 = AssignArg(var_name, right, append, w)
2081	assign_args.append(arg2)
2082
2083	else: # e.g. export $dynamic
2084	argv = self._EvalWordToArgv(w)
2085	for arg in argv:
2086	arg2 = _SplitAssignArg(arg, w)
2087	assign_args.append(arg2)
2088
2089	else:
2090	argv = self._EvalWordToArgv(w)
2091	for arg in argv:
2092	if arg.startswith('-') or arg.startswith('+'):
2093	# e.g. declare -r +r
2094	flags.append(arg)
2095	flag_locs.append(w)
2096
2097	# Shortcut that relies on -f and -F always meaning "function" for
2098	# all assignment builtins
2099	if 'f' in arg or 'F' in arg:
2100	eval_to_pairs = False
2101
2102	else: # e.g. export $dynamic
2103	if eval_to_pairs:
2104	arg2 = _SplitAssignArg(arg, w)
2105	assign_args.append(arg2)
2106	started_pairs = True
2107	else:
2108	flags.append(arg)
2109
2110	return cmd_value.Assign(builtin_id, flags, flag_locs, assign_args)
2111
2112	def SimpleEvalWordSequence2(self, words, allow_assign):
2113	# type: (List[CompoundWord], bool) -> cmd_value_t
2114	"""Simple word evaluation for YSH."""
2115	strs = [] # type: List[str]
2116	locs = [] # type: List[CompoundWord]
2117
2118	for i, w in enumerate(words):
2119	# No globbing in the first arg for command.Simple.
2120	if i == 0 and allow_assign:
2121	strs0 = self._EvalWordToArgv(w) # respects strict-array
2122	if len(strs0) == 1:
2123	arg0 = strs0[0]
2124	builtin_id = consts.LookupAssignBuiltin(arg0)
2125	if builtin_id != consts.NO_INDEX:
2126	# Same logic as legacy word eval, with no splitting
2127	return self._EvalAssignBuiltin(builtin_id, arg0, words)
2128
2129	strs.extend(strs0)
2130	for _ in strs0:
2131	locs.append(w)
2132	continue
2133
2134	if glob_.LooksLikeStaticGlob(w):
2135	val = self.EvalWordToString(w) # respects strict-array
2136	num_appended = self.globber.Expand(val.s, strs)
2137	if num_appended < 0:
2138	raise error.FailGlob('Pattern %r matched no files' % val.s,
2139	w)
2140	for _ in xrange(num_appended):
2141	locs.append(w)
2142	continue
2143
2144	part_vals = [] # type: List[part_value_t]
2145	self._EvalWordToParts(w, part_vals, 0) # not quoted
2146
2147	if 0:
2148	log('')
2149	log('Static: part_vals after _EvalWordToParts:')
2150	for entry in part_vals:
2151	log(' %s', entry)
2152
2153	# Still need to process
2154	frames = _MakeWordFrames(part_vals)
2155
2156	if 0:
2157	log('')
2158	log('Static: frames after _MakeWordFrames:')
2159	for entry in frames:
2160	log(' %s', entry)
2161
2162	# We will still allow x"${a[@]"x, though it's deprecated by @a, which
2163	# disallows such expressions at parse time.
2164	for frame in frames:
2165	if len(frame): # empty array gives empty frame!
2166	tmp = [piece.s for piece in frame]
2167	strs.append(''.join(tmp)) # no split or glob
2168	locs.append(w)
2169
2170	return cmd_value.Argv(strs, locs, None, None, None, None)
2171
2172	def EvalWordSequence2(self, words, allow_assign=False):
2173	# type: (List[CompoundWord], bool) -> cmd_value_t
2174	"""Turns a list of Words into a list of strings.
2175
2176	Unlike the EvalWord*() methods, it does globbing.
2177
2178	Args:
2179	words: list of Word instances
2180
2181	Returns:
2182	argv: list of string arguments, or None if there was an eval error
2183	"""
2184	if self.exec_opts.simple_word_eval():
2185	return self.SimpleEvalWordSequence2(words, allow_assign)
2186
2187	# Parse time:
2188	# 1. brace expansion. TODO: Do at parse time.
2189	# 2. Tilde detection. DONE at parse time. Only if Id.Lit_Tilde is the
2190	# first WordPart.
2191	#
2192	# Run time:
2193	# 3. tilde sub, var sub, command sub, arith sub. These are all
2194	# "concurrent" on WordParts. (optional process sub with <() )
2195	# 4. word splitting. Can turn this off with a shell option? Definitely
2196	# off for oil.
2197	# 5. globbing -- several exec_opts affect this: nullglob, safeglob, etc.
2198
2199	#log('W %s', words)
2200	strs = [] # type: List[str]
2201	locs = [] # type: List[CompoundWord]
2202
2203	n = 0
2204	for i, w in enumerate(words):
2205	fast_str = word_.FastStrEval(w)
2206	if fast_str is not None:
2207	strs.append(fast_str)
2208	locs.append(w)
2209
2210	# e.g. the 'local' in 'local a=b c=d' will be here
2211	if allow_assign and i == 0:
2212	builtin_id = consts.LookupAssignBuiltin(fast_str)
2213	if builtin_id != consts.NO_INDEX:
2214	return self._EvalAssignBuiltin(builtin_id, fast_str,
2215	words)
2216	continue
2217
2218	part_vals = [] # type: List[part_value_t]
2219	self._EvalWordToParts(w, part_vals, EXTGLOB_FILES)
2220
2221	# DYNAMICALLY detect if we're going to run an assignment builtin, and
2222	# change the rest of the evaluation algorithm if so.
2223	#
2224	# We want to allow:
2225	# e=export
2226	# $e foo=bar
2227	#
2228	# But we don't want to evaluate the first word twice in the case of:
2229	# $(some-command) --flag
2230	if allow_assign and i == 0 and len(part_vals) == 1:
2231	val0 = part_vals[0]
2232	UP_val0 = val0
2233	if val0.tag() == part_value_e.String:
2234	val0 = cast(Piece, UP_val0)
2235	if not val0.quoted:
2236	builtin_id = consts.LookupAssignBuiltin(val0.s)
2237	if builtin_id != consts.NO_INDEX:
2238	return self._EvalAssignBuiltin(
2239	builtin_id, val0.s, words)
2240
2241	if 0:
2242	log('')
2243	log('part_vals after _EvalWordToParts:')
2244	for entry in part_vals:
2245	log(' %s', entry)
2246
2247	frames = _MakeWordFrames(part_vals)
2248	if 0:
2249	log('')
2250	log('frames after _MakeWordFrames:')
2251	for entry in frames:
2252	log(' %s', entry)
2253
2254	# Do splitting and globbing. Each frame will append zero or more args.
2255	for frame in frames:
2256	self._EvalWordFrame(frame, strs)
2257
2258	# Fill in locations parallel to strs.
2259	n_next = len(strs)
2260	for _ in xrange(n_next - n):
2261	locs.append(w)
2262	n = n_next
2263
2264	# A non-assignment command.
2265	# NOTE: Can't look up builtins here like we did for assignment, because
2266	# functions can override builtins.
2267	return cmd_value.Argv(strs, locs, None, None, None, None)
2268
2269	def EvalWordSequence(self, words):
2270	# type: (List[CompoundWord]) -> List[str]
2271	"""For arrays and for loops.
2272
2273	They don't allow assignment builtins.
2274	"""
2275	UP_cmd_val = self.EvalWordSequence2(words)
2276
2277	assert UP_cmd_val.tag() == cmd_value_e.Argv
2278	cmd_val = cast(cmd_value.Argv, UP_cmd_val)
2279	return cmd_val.argv
2280
2281
2282	class NormalWordEvaluator(AbstractWordEvaluator):
2283
2284	def __init__(
2285	self,
2286	mem, # type: state.Mem
2287	exec_opts, # type: optview.Exec
2288	mutable_opts, # type: state.MutableOpts
2289	tilde_ev, # type: TildeEvaluator
2290	splitter, # type: SplitContext
2291	errfmt, # type: ErrorFormatter
2292	):
2293	# type: (...) -> None
2294	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2295	tilde_ev, splitter, errfmt)
2296	self.shell_ex = None # type: _Executor
2297
2298	def CheckCircularDeps(self):
2299	# type: () -> None
2300	assert self.arith_ev is not None
2301	# Disabled for pure OSH
2302	#assert self.expr_ev is not None
2303	assert self.shell_ex is not None
2304	assert self.prompt_ev is not None
2305
2306	def _EvalCommandSub(self, cs_part, quoted):
2307	# type: (CommandSub, bool) -> part_value_t
2308	stdout_str = self.shell_ex.RunCommandSub(cs_part)
2309
2310	if cs_part.left_token.id == Id.Left_AtParen:
2311	# YSH splitting algorithm: does not depend on IFS
2312	try:
2313	strs = j8.SplitJ8Lines(stdout_str)
2314	except error.Decode as e:
2315	# status code 4 is special, for encode/decode errors.
2316	raise error.Structured(4, e.Message(), cs_part.left_token)
2317
2318	#strs = self.splitter.SplitForWordEval(stdout_str)
2319	return part_value.Array(strs)
2320	else:
2321	return Piece(stdout_str, quoted, not quoted)
2322
2323	def _EvalProcessSub(self, cs_part):
2324	# type: (CommandSub) -> Piece
2325	dev_path = self.shell_ex.RunProcessSub(cs_part)
2326	# pretend it's quoted; no split or glob
2327	return Piece(dev_path, True, False)
2328
2329
2330	_DUMMY = '__NO_COMMAND_SUB__'
2331
2332
2333	class CompletionWordEvaluator(AbstractWordEvaluator):
2334	"""An evaluator that has no access to an executor.
2335
2336	NOTE: core/completion.py doesn't actually try to use these strings to
2337	complete. If you have something like 'echo $(echo hi)/f<TAB>', it sees the
2338	inner command as the last one, and knows that it is not at the end of the
2339	line.
2340	"""
2341
2342	def __init__(
2343	self,
2344	mem, # type: state.Mem
2345	exec_opts, # type: optview.Exec
2346	mutable_opts, # type: state.MutableOpts
2347	tilde_ev, # type: TildeEvaluator
2348	splitter, # type: SplitContext
2349	errfmt, # type: ErrorFormatter
2350	):
2351	# type: (...) -> None
2352	AbstractWordEvaluator.__init__(self, mem, exec_opts, mutable_opts,
2353	tilde_ev, splitter, errfmt)
2354
2355	def CheckCircularDeps(self):
2356	# type: () -> None
2357	assert self.prompt_ev is not None
2358	assert self.arith_ev is not None
2359	assert self.expr_ev is not None
2360
2361	def _EvalCommandSub(self, cs_part, quoted):
2362	# type: (CommandSub, bool) -> part_value_t
2363	if cs_part.left_token.id == Id.Left_AtParen:
2364	return part_value.Array([_DUMMY])
2365	else:
2366	return Piece(_DUMMY, quoted, not quoted)
2367
2368	def _EvalProcessSub(self, cs_part):
2369	# type: (CommandSub) -> Piece
2370	# pretend it's quoted; no split or glob
2371	return Piece('__NO_PROCESS_SUB__', True, False)
2372
2373
2374	# vim: sw=4