OILS / builtin / func_misc.py View on Github | oilshell.org

790 lines, 456 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj)
9
10from core import error
11from core import num
12from core import state
13from display import pp_value
14from display import ui
15from core import vm
16from data_lang import j8
17from frontend import match
18from frontend import typed_args
19from mycpp import mops
20from mycpp import mylib
21from mycpp.mylib import NewDict, iteritems, log, tagswitch
22from ysh import expr_eval
23from ysh import val_ops
24
25from typing import TYPE_CHECKING, Dict, List, Optional, cast
26if TYPE_CHECKING:
27 from osh import glob_
28 from osh import split
29
30_ = log
31
32
33class Object(vm._Callable):
34 """Create a value.Obj
35
36 The order of params follows JavaScript's Object.create():
37 var obj = Object(prototype, props)
38 """
39
40 def __init__(self):
41 # type: () -> None
42 pass
43
44 def Call(self, rd):
45 # type: (typed_args.Reader) -> value_t
46
47 prototype = rd.PosValue()
48 props = rd.PosDict()
49 rd.Done()
50
51 chain = None # type: Optional[Obj]
52 UP_prototype = prototype
53 with tagswitch(prototype) as case:
54 if case(value_e.Null):
55 pass
56 elif case(value_e.Obj):
57 prototype = cast(Obj, UP_prototype)
58 chain = prototype
59 else:
60 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
61 rd.BlamePos())
62
63 return Obj(chain, props)
64
65
66class Prototype(vm._Callable):
67 """Get an object's prototype."""
68
69 def __init__(self):
70 # type: () -> None
71 pass
72
73 def Call(self, rd):
74 # type: (typed_args.Reader) -> value_t
75 obj = rd.PosObj()
76 rd.Done()
77
78 if obj.prototype is None:
79 return value.Null
80
81 return obj.prototype
82
83
84class PropView(vm._Callable):
85 """Get a Dict view of an object's properties."""
86
87 def __init__(self):
88 # type: () -> None
89 pass
90
91 def Call(self, rd):
92 # type: (typed_args.Reader) -> value_t
93 obj = rd.PosObj()
94 rd.Done()
95
96 return value.Dict(obj.d)
97
98
99class Len(vm._Callable):
100
101 def __init__(self):
102 # type: () -> None
103 pass
104
105 def Call(self, rd):
106 # type: (typed_args.Reader) -> value_t
107
108 x = rd.PosValue()
109 rd.Done()
110
111 UP_x = x
112 with tagswitch(x) as case:
113 if case(value_e.List):
114 x = cast(value.List, UP_x)
115 return num.ToBig(len(x.items))
116
117 elif case(value_e.Dict):
118 x = cast(value.Dict, UP_x)
119 return num.ToBig(len(x.d))
120
121 elif case(value_e.Str):
122 x = cast(value.Str, UP_x)
123 return num.ToBig(len(x.s))
124
125 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
126 rd.BlamePos())
127
128
129class Type(vm._Callable):
130
131 def __init__(self):
132 # type: () -> None
133 pass
134
135 def Call(self, rd):
136 # type: (typed_args.Reader) -> value_t
137
138 val = rd.PosValue()
139 rd.Done()
140
141 return value.Str(ui.ValType(val))
142
143
144class Join(vm._Callable):
145 """Both free function join() and List->join() method."""
146
147 def __init__(self):
148 # type: () -> None
149 pass
150
151 def Call(self, rd):
152 # type: (typed_args.Reader) -> value_t
153
154 li = rd.PosList()
155 delim = rd.OptionalStr(default_='')
156 rd.Done()
157
158 strs = [] # type: List[str]
159 for i, el in enumerate(li):
160 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
161
162 return value.Str(delim.join(strs))
163
164
165class Maybe(vm._Callable):
166
167 def __init__(self):
168 # type: () -> None
169 pass
170
171 def Call(self, rd):
172 # type: (typed_args.Reader) -> value_t
173
174 val = rd.PosValue()
175 rd.Done()
176
177 if val == value.Null:
178 return value.List([])
179
180 s = val_ops.ToStr(
181 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
182 rd.LeftParenToken())
183 if len(s):
184 return value.List([val]) # use val to avoid needlessly copy
185
186 return value.List([])
187
188
189class Bool(vm._Callable):
190
191 def __init__(self):
192 # type: () -> None
193 pass
194
195 def Call(self, rd):
196 # type: (typed_args.Reader) -> value_t
197
198 val = rd.PosValue()
199 rd.Done()
200
201 return value.Bool(val_ops.ToBool(val))
202
203
204class Int(vm._Callable):
205
206 def __init__(self):
207 # type: () -> None
208 pass
209
210 def Call(self, rd):
211 # type: (typed_args.Reader) -> value_t
212
213 val = rd.PosValue()
214 rd.Done()
215
216 UP_val = val
217 with tagswitch(val) as case:
218 if case(value_e.Int):
219 return val
220
221 elif case(value_e.Bool):
222 val = cast(value.Bool, UP_val)
223 return value.Int(mops.FromBool(val.b))
224
225 elif case(value_e.Float):
226 val = cast(value.Float, UP_val)
227 ok, big_int = mops.FromFloat(val.f)
228 if ok:
229 return value.Int(big_int)
230 else:
231 raise error.Expr(
232 "Can't convert float %s to Int" %
233 pp_value.FloatString(val.f), rd.BlamePos())
234
235 elif case(value_e.Str):
236 val = cast(value.Str, UP_val)
237 if not match.LooksLikeInteger(val.s):
238 raise error.Expr("Can't convert %s to Int" % val.s,
239 rd.BlamePos())
240
241 return value.Int(mops.FromStr(val.s))
242
243 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
244 rd.BlamePos())
245
246
247class Float(vm._Callable):
248
249 def __init__(self):
250 # type: () -> None
251 pass
252
253 def Call(self, rd):
254 # type: (typed_args.Reader) -> value_t
255
256 val = rd.PosValue()
257 rd.Done()
258
259 UP_val = val
260 with tagswitch(val) as case:
261 if case(value_e.Int):
262 val = cast(value.Int, UP_val)
263 return value.Float(mops.ToFloat(val.i))
264
265 elif case(value_e.Float):
266 return val
267
268 elif case(value_e.Str):
269 val = cast(value.Str, UP_val)
270 if not match.LooksLikeFloat(val.s):
271 raise error.Expr('Cannot convert %s to Float' % val.s,
272 rd.BlamePos())
273
274 return value.Float(float(val.s))
275
276 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
277 rd.BlamePos())
278
279
280class Str_(vm._Callable):
281
282 def __init__(self):
283 # type: () -> None
284 pass
285
286 def Call(self, rd):
287 # type: (typed_args.Reader) -> value_t
288
289 val = rd.PosValue()
290 rd.Done()
291
292 # TODO: Should we call Stringify here? That would handle Eggex.
293
294 UP_val = val
295 with tagswitch(val) as case:
296 if case(value_e.Int):
297 val = cast(value.Int, UP_val)
298 return value.Str(mops.ToStr(val.i))
299
300 elif case(value_e.Float):
301 val = cast(value.Float, UP_val)
302 return value.Str(str(val.f))
303
304 elif case(value_e.Str):
305 return val
306
307 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
308 rd.BlamePos())
309
310
311class List_(vm._Callable):
312
313 def __init__(self):
314 # type: () -> None
315 pass
316
317 def Call(self, rd):
318 # type: (typed_args.Reader) -> value_t
319
320 val = rd.PosValue()
321 rd.Done()
322
323 l = [] # type: List[value_t]
324 it = None # type: val_ops.Iterator
325 UP_val = val
326 with tagswitch(val) as case:
327 if case(value_e.List):
328 val = cast(value.List, UP_val)
329 it = val_ops.ListIterator(val)
330
331 elif case(value_e.Dict):
332 val = cast(value.Dict, UP_val)
333 it = val_ops.DictIterator(val)
334
335 elif case(value_e.Range):
336 val = cast(value.Range, UP_val)
337 it = val_ops.RangeIterator(val)
338
339 else:
340 raise error.TypeErr(val,
341 'list() expected Dict, List, or Range',
342 rd.BlamePos())
343
344 assert it is not None
345 while True:
346 first = it.FirstValue()
347 if first is None:
348 break
349 l.append(first)
350 it.Next()
351
352 return value.List(l)
353
354
355class DictFunc(vm._Callable):
356
357 def __init__(self):
358 # type: () -> None
359 pass
360
361 def Call(self, rd):
362 # type: (typed_args.Reader) -> value_t
363
364 val = rd.PosValue()
365 rd.Done()
366
367 UP_val = val
368 with tagswitch(val) as case:
369 if case(value_e.Dict):
370 d = NewDict() # type: Dict[str, value_t]
371 val = cast(value.Dict, UP_val)
372 for k, v in iteritems(val.d):
373 d[k] = v
374
375 return value.Dict(d)
376
377 elif case(value_e.Obj):
378 d = NewDict()
379 val = cast(Obj, UP_val)
380 for k, v in iteritems(val.d):
381 d[k] = v
382
383 return value.Dict(d)
384
385 elif case(value_e.BashAssoc):
386 d = NewDict()
387 val = cast(value.BashAssoc, UP_val)
388 for k, s in iteritems(val.d):
389 d[k] = value.Str(s)
390
391 return value.Dict(d)
392
393 raise error.TypeErr(val, 'dict() expected Dict, Obj, or BashAssoc',
394 rd.BlamePos())
395
396
397class Runes(vm._Callable):
398
399 def __init__(self):
400 # type: () -> None
401 pass
402
403 def Call(self, rd):
404 # type: (typed_args.Reader) -> value_t
405 return value.Null
406
407
408class EncodeRunes(vm._Callable):
409
410 def __init__(self):
411 # type: () -> None
412 pass
413
414 def Call(self, rd):
415 # type: (typed_args.Reader) -> value_t
416 return value.Null
417
418
419class Bytes(vm._Callable):
420
421 def __init__(self):
422 # type: () -> None
423 pass
424
425 def Call(self, rd):
426 # type: (typed_args.Reader) -> value_t
427 return value.Null
428
429
430class EncodeBytes(vm._Callable):
431
432 def __init__(self):
433 # type: () -> None
434 pass
435
436 def Call(self, rd):
437 # type: (typed_args.Reader) -> value_t
438 return value.Null
439
440
441class Split(vm._Callable):
442
443 def __init__(self, splitter):
444 # type: (split.SplitContext) -> None
445 vm._Callable.__init__(self)
446 self.splitter = splitter
447
448 def Call(self, rd):
449 # type: (typed_args.Reader) -> value_t
450 s = rd.PosStr()
451
452 ifs = rd.OptionalStr()
453
454 rd.Done()
455
456 l = [
457 value.Str(elem)
458 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
459 ] # type: List[value_t]
460 return value.List(l)
461
462
463class FloatsEqual(vm._Callable):
464
465 def __init__(self):
466 # type: () -> None
467 pass
468
469 def Call(self, rd):
470 # type: (typed_args.Reader) -> value_t
471 left = rd.PosFloat()
472 right = rd.PosFloat()
473 rd.Done()
474
475 return value.Bool(left == right)
476
477
478class Glob(vm._Callable):
479
480 def __init__(self, globber):
481 # type: (glob_.Globber) -> None
482 vm._Callable.__init__(self)
483 self.globber = globber
484
485 def Call(self, rd):
486 # type: (typed_args.Reader) -> value_t
487 s = rd.PosStr()
488 rd.Done()
489
490 out = [] # type: List[str]
491 self.globber._Glob(s, out)
492
493 l = [value.Str(elem) for elem in out] # type: List[value_t]
494 return value.List(l)
495
496
497class Shvar_get(vm._Callable):
498 """Look up with dynamic scope."""
499
500 def __init__(self, mem):
501 # type: (state.Mem) -> None
502 vm._Callable.__init__(self)
503 self.mem = mem
504
505 def Call(self, rd):
506 # type: (typed_args.Reader) -> value_t
507 name = rd.PosStr()
508 rd.Done()
509 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
510
511
512class GetVar(vm._Callable):
513 """Look up normal scoping rules."""
514
515 def __init__(self, mem):
516 # type: (state.Mem) -> None
517 vm._Callable.__init__(self)
518 self.mem = mem
519
520 def Call(self, rd):
521 # type: (typed_args.Reader) -> value_t
522 name = rd.PosStr()
523 rd.Done()
524 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
525
526
527class EvalExpr(vm._Callable):
528
529 def __init__(self, expr_ev):
530 # type: (expr_eval.ExprEvaluator) -> None
531 self.expr_ev = expr_ev
532
533 def Call(self, rd):
534 # type: (typed_args.Reader) -> value_t
535 lazy = rd.PosExpr()
536 rd.Done()
537
538 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
539
540 return result
541
542
543class ToJson8(vm._Callable):
544
545 def __init__(self, is_j8):
546 # type: (bool) -> None
547 self.is_j8 = is_j8
548
549 def Call(self, rd):
550 # type: (typed_args.Reader) -> value_t
551
552 val = rd.PosValue()
553 space = mops.BigTruncate(rd.NamedInt('space', 0))
554 rd.Done()
555
556 # Convert from external JS-like API to internal API.
557 if space <= 0:
558 indent = -1
559 else:
560 indent = space
561
562 buf = mylib.BufWriter()
563 try:
564 if self.is_j8:
565 j8.PrintMessage(val, buf, indent)
566 else:
567 j8.PrintJsonMessage(val, buf, indent)
568 except error.Encode as e:
569 # status code 4 is special, for encode/decode errors.
570 raise error.Structured(4, e.Message(), rd.LeftParenToken())
571
572 return value.Str(buf.getvalue())
573
574
575class FromJson8(vm._Callable):
576
577 def __init__(self, is_j8):
578 # type: (bool) -> None
579 self.is_j8 = is_j8
580
581 def Call(self, rd):
582 # type: (typed_args.Reader) -> value_t
583
584 s = rd.PosStr()
585 rd.Done()
586
587 p = j8.Parser(s, self.is_j8)
588 try:
589 val = p.ParseValue()
590 except error.Decode as e:
591 # Right now I'm not exposing the original string, because that
592 # could lead to a memory leak in the _error Dict.
593 # The message quotes part of the string, and we could improve
594 # that. We could have a substring with context.
595 props = {
596 'start_pos': num.ToBig(e.start_pos),
597 'end_pos': num.ToBig(e.end_pos),
598 } # type: Dict[str, value_t]
599 # status code 4 is special, for encode/decode errors.
600 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
601
602 return val
603
604
605class BashArrayToSparse(vm._Callable):
606 """
607 value.BashArray -> value.SparseArray, for testing
608 """
609
610 def __init__(self):
611 # type: () -> None
612 pass
613
614 def Call(self, rd):
615 # type: (typed_args.Reader) -> value_t
616
617 strs = rd.PosBashArray()
618 rd.Done()
619
620 d = {} # type: Dict[mops.BigInt, str]
621 max_index = mops.MINUS_ONE # max index for empty array
622 for i, s in enumerate(strs):
623 if s is not None:
624 big_i = mops.IntWiden(i)
625 d[big_i] = s
626 if mops.Greater(big_i, max_index):
627 max_index = big_i
628
629 return value.SparseArray(d, max_index)
630
631
632class SparseOp(vm._Callable):
633 """
634 All ops on value.SparseArray, for testing performance
635 """
636
637 def __init__(self):
638 # type: () -> None
639 pass
640
641 def Call(self, rd):
642 # type: (typed_args.Reader) -> value_t
643
644 sp = rd.PosSparseArray()
645 d = sp.d
646 #i = mops.BigTruncate(rd.PosInt())
647 op_name = rd.PosStr()
648
649 no_str = None # type: str
650
651 if op_name == 'len': # ${#a[@]}
652 rd.Done()
653 return num.ToBig(len(d))
654
655 elif op_name == 'get': # ${a[42]}
656 index = rd.PosInt()
657 rd.Done()
658
659 s = d.get(index)
660 if s is None:
661 return value.Null
662 else:
663 return value.Str(s)
664
665 elif op_name == 'set': # a[42]=foo
666 index = rd.PosInt()
667 s = rd.PosStr()
668 rd.Done()
669
670 d[index] = s
671
672 if mops.Greater(index, sp.max_index):
673 sp.max_index = index
674
675 return value.Int(mops.ZERO)
676
677 elif op_name == 'unset': # unset 'a[1]'
678 index = rd.PosInt()
679 rd.Done()
680
681 mylib.dict_erase(d, index)
682
683 max_index = mops.MINUS_ONE # Note: this works if d is not empty
684 for i1 in d:
685 if mops.Greater(i1, max_index): # i1 > max_index
686 max_index = i1
687 sp.max_index = max_index
688
689 return value.Int(mops.ZERO)
690
691 elif op_name == 'subst': # "${a[@]}"
692 # Algorithm to expand a Dict[BigInt, Str]
693 #
694 # 1. Copy the integer keys into a new List
695 # 2. Sort them in numeric order
696 # 3. Create a List[str] that's the same size as the keys
697 # 4. Loop through sorted keys, look up value, and populate list
698 #
699 # There is another possible algorithm:
700 #
701 # 1. Copy the VALUES into a new list
702 # 2. Somehow sort them by the CORRESPONDING key, which depends on
703 # Slab<> POSITION. I think this does not fit within the
704 # std::sort() model. I think we would have to write a little custom
705 # sort algorithm.
706
707 keys = d.keys()
708 mylib.BigIntSort(keys)
709 # Pre-allocate
710 items = [no_str] * len(d) # type: List[str]
711 j = 0
712 for i in keys:
713 s = d.get(i)
714 assert s is not None
715 items[j] = s
716 j += 1
717 return value.BashArray(items)
718
719 elif op_name == 'keys': # "${!a[@]}"
720 keys = d.keys()
721 mylib.BigIntSort(keys)
722 items = [mops.ToStr(k) for k in keys]
723
724 # TODO: return SparseArray
725 return value.BashArray(items)
726
727 elif op_name == 'slice': # "${a[@]:0:5}"
728 start = rd.PosInt()
729 end = rd.PosInt()
730 rd.Done()
731
732 n = mops.BigTruncate(mops.Sub(end, start))
733 #log('start %d - end %d', start.i, end.i)
734
735 # Pre-allocate
736 items2 = [no_str] * n # type: List[str]
737
738 # Iterate from start to end. Note that this algorithm is
739 # theoretically slower than bash in the case where the array is
740 # sparse (in the part selected by the slice)
741 #
742 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
743 # elements, OSH will iterate through 999 integers and do 999 dict
744 # lookups, while bash will follow 3 pointers.
745 #
746 # However, in practice, I think iterating through integers is
747 # cheap.
748
749 j = 0
750 i = start
751 while mops.Greater(end, i): # i < end
752 s = d.get(i)
753 #log('s %s', s)
754 if s is not None:
755 items2[j] = s
756 j += 1
757
758 i = mops.Add(i, mops.ONE) # i += 1
759
760 # TODO: return SparseArray
761 return value.BashArray(items2)
762
763 elif op_name == 'append': # a+=(x y)
764 strs = rd.PosBashArray()
765
766 # TODO: We can maintain the max index in the value.SparseArray(),
767 # so that it's O(1) to append rather than O(n)
768 # - Update on 'set' is O(1)
769 # - Update on 'unset' is potentially O(n)
770
771 if 0:
772 max_index = mops.MINUS_ONE # Note: this works for empty arrays
773 for i1 in d:
774 if mops.Greater(i1, max_index): # i1 > max_index
775 max_index = i1
776 else:
777 max_index = sp.max_index
778
779 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
780 for s in strs:
781 d[i2] = s
782 i2 = mops.Add(i2, mops.ONE) # i2 += 1
783
784 # sp.max_index += len(strs)
785 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
786 return value.Int(mops.ZERO)
787
788 else:
789 print('Invalid SparseArray operation %r' % op_name)
790 return value.Int(mops.ZERO)