OILS / builtin / func_misc.py View on Github | oilshell.org

716 lines, 415 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str)
9
10from core import error
11from core import num
12from core import state
13from display import pp_value
14from display import ui
15from core import vm
16from data_lang import j8
17from frontend import match
18from frontend import typed_args
19from mycpp import mops
20from mycpp import mylib
21from mycpp.mylib import NewDict, iteritems, log, tagswitch
22from ysh import expr_eval
23from ysh import val_ops
24
25from typing import TYPE_CHECKING, Dict, List, cast
26if TYPE_CHECKING:
27 from osh import glob_
28 from osh import split
29
30_ = log
31
32
33class Len(vm._Callable):
34
35 def __init__(self):
36 # type: () -> None
37 pass
38
39 def Call(self, rd):
40 # type: (typed_args.Reader) -> value_t
41
42 x = rd.PosValue()
43 rd.Done()
44
45 UP_x = x
46 with tagswitch(x) as case:
47 if case(value_e.List):
48 x = cast(value.List, UP_x)
49 return num.ToBig(len(x.items))
50
51 elif case(value_e.Dict):
52 x = cast(value.Dict, UP_x)
53 return num.ToBig(len(x.d))
54
55 elif case(value_e.Str):
56 x = cast(value.Str, UP_x)
57 return num.ToBig(len(x.s))
58
59 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
60 rd.BlamePos())
61
62
63class Type(vm._Callable):
64
65 def __init__(self):
66 # type: () -> None
67 pass
68
69 def Call(self, rd):
70 # type: (typed_args.Reader) -> value_t
71
72 val = rd.PosValue()
73 rd.Done()
74
75 return value.Str(ui.ValType(val))
76
77
78class Join(vm._Callable):
79 """Both free function join() and List->join() method."""
80
81 def __init__(self):
82 # type: () -> None
83 pass
84
85 def Call(self, rd):
86 # type: (typed_args.Reader) -> value_t
87
88 li = rd.PosList()
89 delim = rd.OptionalStr(default_='')
90 rd.Done()
91
92 strs = [] # type: List[str]
93 for i, el in enumerate(li):
94 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
95
96 return value.Str(delim.join(strs))
97
98
99class Maybe(vm._Callable):
100
101 def __init__(self):
102 # type: () -> None
103 pass
104
105 def Call(self, rd):
106 # type: (typed_args.Reader) -> value_t
107
108 val = rd.PosValue()
109 rd.Done()
110
111 if val == value.Null:
112 return value.List([])
113
114 s = val_ops.ToStr(
115 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
116 rd.LeftParenToken())
117 if len(s):
118 return value.List([val]) # use val to avoid needlessly copy
119
120 return value.List([])
121
122
123class Bool(vm._Callable):
124
125 def __init__(self):
126 # type: () -> None
127 pass
128
129 def Call(self, rd):
130 # type: (typed_args.Reader) -> value_t
131
132 val = rd.PosValue()
133 rd.Done()
134
135 return value.Bool(val_ops.ToBool(val))
136
137
138class Int(vm._Callable):
139
140 def __init__(self):
141 # type: () -> None
142 pass
143
144 def Call(self, rd):
145 # type: (typed_args.Reader) -> value_t
146
147 val = rd.PosValue()
148 rd.Done()
149
150 UP_val = val
151 with tagswitch(val) as case:
152 if case(value_e.Int):
153 return val
154
155 elif case(value_e.Bool):
156 val = cast(value.Bool, UP_val)
157 return value.Int(mops.FromBool(val.b))
158
159 elif case(value_e.Float):
160 val = cast(value.Float, UP_val)
161 ok, big_int = mops.FromFloat(val.f)
162 if ok:
163 return value.Int(big_int)
164 else:
165 raise error.Expr(
166 "Can't convert float %s to Int" %
167 pp_value.FloatString(val.f), rd.BlamePos())
168
169 elif case(value_e.Str):
170 val = cast(value.Str, UP_val)
171 if not match.LooksLikeInteger(val.s):
172 raise error.Expr("Can't convert %s to Int" % val.s,
173 rd.BlamePos())
174
175 return value.Int(mops.FromStr(val.s))
176
177 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
178 rd.BlamePos())
179
180
181class Float(vm._Callable):
182
183 def __init__(self):
184 # type: () -> None
185 pass
186
187 def Call(self, rd):
188 # type: (typed_args.Reader) -> value_t
189
190 val = rd.PosValue()
191 rd.Done()
192
193 UP_val = val
194 with tagswitch(val) as case:
195 if case(value_e.Int):
196 val = cast(value.Int, UP_val)
197 return value.Float(mops.ToFloat(val.i))
198
199 elif case(value_e.Float):
200 return val
201
202 elif case(value_e.Str):
203 val = cast(value.Str, UP_val)
204 if not match.LooksLikeFloat(val.s):
205 raise error.Expr('Cannot convert %s to Float' % val.s,
206 rd.BlamePos())
207
208 return value.Float(float(val.s))
209
210 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
211 rd.BlamePos())
212
213
214class Str_(vm._Callable):
215
216 def __init__(self):
217 # type: () -> None
218 pass
219
220 def Call(self, rd):
221 # type: (typed_args.Reader) -> value_t
222
223 val = rd.PosValue()
224 rd.Done()
225
226 # TODO: Should we call Stringify here? That would handle Eggex.
227
228 UP_val = val
229 with tagswitch(val) as case:
230 if case(value_e.Int):
231 val = cast(value.Int, UP_val)
232 return value.Str(mops.ToStr(val.i))
233
234 elif case(value_e.Float):
235 val = cast(value.Float, UP_val)
236 return value.Str(str(val.f))
237
238 elif case(value_e.Str):
239 return val
240
241 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
242 rd.BlamePos())
243
244
245class List_(vm._Callable):
246
247 def __init__(self):
248 # type: () -> None
249 pass
250
251 def Call(self, rd):
252 # type: (typed_args.Reader) -> value_t
253
254 val = rd.PosValue()
255 rd.Done()
256
257 l = [] # type: List[value_t]
258 it = None # type: val_ops.Iterator
259 UP_val = val
260 with tagswitch(val) as case:
261 if case(value_e.List):
262 val = cast(value.List, UP_val)
263 it = val_ops.ListIterator(val)
264
265 elif case(value_e.Dict):
266 val = cast(value.Dict, UP_val)
267 it = val_ops.DictIterator(val)
268
269 elif case(value_e.Range):
270 val = cast(value.Range, UP_val)
271 it = val_ops.RangeIterator(val)
272
273 else:
274 raise error.TypeErr(val,
275 'list() expected Dict, List, or Range',
276 rd.BlamePos())
277
278 assert it is not None
279 while True:
280 first = it.FirstValue()
281 if first is None:
282 break
283 l.append(first)
284 it.Next()
285
286 return value.List(l)
287
288
289class Dict_(vm._Callable):
290
291 def __init__(self):
292 # type: () -> None
293 pass
294
295 def Call(self, rd):
296 # type: (typed_args.Reader) -> value_t
297
298 val = rd.PosValue()
299 rd.Done()
300
301 UP_val = val
302 with tagswitch(val) as case:
303 if case(value_e.Dict):
304 d = NewDict() # type: Dict[str, value_t]
305 val = cast(value.Dict, UP_val)
306 for k, v in iteritems(val.d):
307 d[k] = v
308
309 return value.Dict(d)
310
311 elif case(value_e.BashAssoc):
312 d = NewDict()
313 val = cast(value.BashAssoc, UP_val)
314 for k, s in iteritems(val.d):
315 d[k] = value.Str(s)
316
317 return value.Dict(d)
318
319 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
320 rd.BlamePos())
321
322
323class Runes(vm._Callable):
324
325 def __init__(self):
326 # type: () -> None
327 pass
328
329 def Call(self, rd):
330 # type: (typed_args.Reader) -> value_t
331 return value.Null
332
333
334class EncodeRunes(vm._Callable):
335
336 def __init__(self):
337 # type: () -> None
338 pass
339
340 def Call(self, rd):
341 # type: (typed_args.Reader) -> value_t
342 return value.Null
343
344
345class Bytes(vm._Callable):
346
347 def __init__(self):
348 # type: () -> None
349 pass
350
351 def Call(self, rd):
352 # type: (typed_args.Reader) -> value_t
353 return value.Null
354
355
356class EncodeBytes(vm._Callable):
357
358 def __init__(self):
359 # type: () -> None
360 pass
361
362 def Call(self, rd):
363 # type: (typed_args.Reader) -> value_t
364 return value.Null
365
366
367class Split(vm._Callable):
368
369 def __init__(self, splitter):
370 # type: (split.SplitContext) -> None
371 vm._Callable.__init__(self)
372 self.splitter = splitter
373
374 def Call(self, rd):
375 # type: (typed_args.Reader) -> value_t
376 s = rd.PosStr()
377
378 ifs = rd.OptionalStr()
379
380 rd.Done()
381
382 l = [
383 value.Str(elem)
384 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
385 ] # type: List[value_t]
386 return value.List(l)
387
388
389class FloatsEqual(vm._Callable):
390
391 def __init__(self):
392 # type: () -> None
393 pass
394
395 def Call(self, rd):
396 # type: (typed_args.Reader) -> value_t
397 left = rd.PosFloat()
398 right = rd.PosFloat()
399 rd.Done()
400
401 return value.Bool(left == right)
402
403
404class Glob(vm._Callable):
405
406 def __init__(self, globber):
407 # type: (glob_.Globber) -> None
408 vm._Callable.__init__(self)
409 self.globber = globber
410
411 def Call(self, rd):
412 # type: (typed_args.Reader) -> value_t
413 s = rd.PosStr()
414 rd.Done()
415
416 out = [] # type: List[str]
417 self.globber._Glob(s, out)
418
419 l = [value.Str(elem) for elem in out] # type: List[value_t]
420 return value.List(l)
421
422
423class Shvar_get(vm._Callable):
424 """Look up with dynamic scope."""
425
426 def __init__(self, mem):
427 # type: (state.Mem) -> None
428 vm._Callable.__init__(self)
429 self.mem = mem
430
431 def Call(self, rd):
432 # type: (typed_args.Reader) -> value_t
433 name = rd.PosStr()
434 rd.Done()
435 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
436
437
438class GetVar(vm._Callable):
439 """Look up normal scoping rules."""
440
441 def __init__(self, mem):
442 # type: (state.Mem) -> None
443 vm._Callable.__init__(self)
444 self.mem = mem
445
446 def Call(self, rd):
447 # type: (typed_args.Reader) -> value_t
448 name = rd.PosStr()
449 rd.Done()
450 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
451
452
453class EvalExpr(vm._Callable):
454
455 def __init__(self, expr_ev):
456 # type: (expr_eval.ExprEvaluator) -> None
457 self.expr_ev = expr_ev
458
459 def Call(self, rd):
460 # type: (typed_args.Reader) -> value_t
461 lazy = rd.PosExpr()
462 rd.Done()
463
464 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
465
466 return result
467
468
469class ToJson8(vm._Callable):
470
471 def __init__(self, is_j8):
472 # type: (bool) -> None
473 self.is_j8 = is_j8
474
475 def Call(self, rd):
476 # type: (typed_args.Reader) -> value_t
477
478 val = rd.PosValue()
479 space = mops.BigTruncate(rd.NamedInt('space', 0))
480 rd.Done()
481
482 # Convert from external JS-like API to internal API.
483 if space <= 0:
484 indent = -1
485 else:
486 indent = space
487
488 buf = mylib.BufWriter()
489 try:
490 if self.is_j8:
491 j8.PrintMessage(val, buf, indent)
492 else:
493 j8.PrintJsonMessage(val, buf, indent)
494 except error.Encode as e:
495 # status code 4 is special, for encode/decode errors.
496 raise error.Structured(4, e.Message(), rd.LeftParenToken())
497
498 return value.Str(buf.getvalue())
499
500
501class FromJson8(vm._Callable):
502
503 def __init__(self, is_j8):
504 # type: (bool) -> None
505 self.is_j8 = is_j8
506
507 def Call(self, rd):
508 # type: (typed_args.Reader) -> value_t
509
510 s = rd.PosStr()
511 rd.Done()
512
513 p = j8.Parser(s, self.is_j8)
514 try:
515 val = p.ParseValue()
516 except error.Decode as e:
517 # Right now I'm not exposing the original string, because that
518 # could lead to a memory leak in the _error Dict.
519 # The message quotes part of the string, and we could improve
520 # that. We could have a substring with context.
521 props = {
522 'start_pos': num.ToBig(e.start_pos),
523 'end_pos': num.ToBig(e.end_pos),
524 } # type: Dict[str, value_t]
525 # status code 4 is special, for encode/decode errors.
526 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
527
528 return val
529
530
531class BashArrayToSparse(vm._Callable):
532 """
533 value.BashArray -> value.SparseArray, for testing
534 """
535
536 def __init__(self):
537 # type: () -> None
538 pass
539
540 def Call(self, rd):
541 # type: (typed_args.Reader) -> value_t
542
543 strs = rd.PosBashArray()
544 rd.Done()
545
546 d = {} # type: Dict[mops.BigInt, str]
547 max_index = mops.MINUS_ONE # max index for empty array
548 for i, s in enumerate(strs):
549 if s is not None:
550 big_i = mops.IntWiden(i)
551 d[big_i] = s
552 if mops.Greater(big_i, max_index):
553 max_index = big_i
554
555 return value.SparseArray(d, max_index)
556
557
558class SparseOp(vm._Callable):
559 """
560 All ops on value.SparseArray, for testing performance
561 """
562
563 def __init__(self):
564 # type: () -> None
565 pass
566
567 def Call(self, rd):
568 # type: (typed_args.Reader) -> value_t
569
570 sp = rd.PosSparseArray()
571 d = sp.d
572 #i = mops.BigTruncate(rd.PosInt())
573 op_name = rd.PosStr()
574
575 no_str = None # type: str
576
577 if op_name == 'len': # ${#a[@]}
578 rd.Done()
579 return num.ToBig(len(d))
580
581 elif op_name == 'get': # ${a[42]}
582 index = rd.PosInt()
583 rd.Done()
584
585 s = d.get(index)
586 if s is None:
587 return value.Null
588 else:
589 return value.Str(s)
590
591 elif op_name == 'set': # a[42]=foo
592 index = rd.PosInt()
593 s = rd.PosStr()
594 rd.Done()
595
596 d[index] = s
597
598 if mops.Greater(index, sp.max_index):
599 sp.max_index = index
600
601 return value.Int(mops.ZERO)
602
603 elif op_name == 'unset': # unset 'a[1]'
604 index = rd.PosInt()
605 rd.Done()
606
607 mylib.dict_erase(d, index)
608
609 max_index = mops.MINUS_ONE # Note: this works if d is not empty
610 for i1 in d:
611 if mops.Greater(i1, max_index): # i1 > max_index
612 max_index = i1
613 sp.max_index = max_index
614
615 return value.Int(mops.ZERO)
616
617 elif op_name == 'subst': # "${a[@]}"
618 # Algorithm to expand a Dict[BigInt, Str]
619 #
620 # 1. Copy the integer keys into a new List
621 # 2. Sort them in numeric order
622 # 3. Create a List[str] that's the same size as the keys
623 # 4. Loop through sorted keys, look up value, and populate list
624 #
625 # There is another possible algorithm:
626 #
627 # 1. Copy the VALUES into a new list
628 # 2. Somehow sort them by the CORRESPONDING key, which depends on
629 # Slab<> POSITION. I think this does not fit within the
630 # std::sort() model. I think we would have to write a little custom
631 # sort algorithm.
632
633 keys = d.keys()
634 mylib.BigIntSort(keys)
635 # Pre-allocate
636 items = [no_str] * len(d) # type: List[str]
637 j = 0
638 for i in keys:
639 s = d.get(i)
640 assert s is not None
641 items[j] = s
642 j += 1
643 return value.BashArray(items)
644
645 elif op_name == 'keys': # "${!a[@]}"
646 keys = d.keys()
647 mylib.BigIntSort(keys)
648 items = [mops.ToStr(k) for k in keys]
649
650 # TODO: return SparseArray
651 return value.BashArray(items)
652
653 elif op_name == 'slice': # "${a[@]:0:5}"
654 start = rd.PosInt()
655 end = rd.PosInt()
656 rd.Done()
657
658 n = mops.BigTruncate(mops.Sub(end, start))
659 #log('start %d - end %d', start.i, end.i)
660
661 # Pre-allocate
662 items2 = [no_str] * n # type: List[str]
663
664 # Iterate from start to end. Note that this algorithm is
665 # theoretically slower than bash in the case where the array is
666 # sparse (in the part selected by the slice)
667 #
668 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
669 # elements, OSH will iterate through 999 integers and do 999 dict
670 # lookups, while bash will follow 3 pointers.
671 #
672 # However, in practice, I think iterating through integers is
673 # cheap.
674
675 j = 0
676 i = start
677 while mops.Greater(end, i): # i < end
678 s = d.get(i)
679 #log('s %s', s)
680 if s is not None:
681 items2[j] = s
682 j += 1
683
684 i = mops.Add(i, mops.ONE) # i += 1
685
686 # TODO: return SparseArray
687 return value.BashArray(items2)
688
689 elif op_name == 'append': # a+=(x y)
690 strs = rd.PosBashArray()
691
692 # TODO: We can maintain the max index in the value.SparseArray(),
693 # so that it's O(1) to append rather than O(n)
694 # - Update on 'set' is O(1)
695 # - Update on 'unset' is potentially O(n)
696
697 if 0:
698 max_index = mops.MINUS_ONE # Note: this works for empty arrays
699 for i1 in d:
700 if mops.Greater(i1, max_index): # i1 > max_index
701 max_index = i1
702 else:
703 max_index = sp.max_index
704
705 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
706 for s in strs:
707 d[i2] = s
708 i2 = mops.Add(i2, mops.ONE) # i2 += 1
709
710 # sp.max_index += len(strs)
711 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
712 return value.Int(mops.ZERO)
713
714 else:
715 print('Invalid SparseArray operation %r' % op_name)
716 return value.Int(mops.ZERO)