OILS / builtin / func_misc.py View on Github | oilshell.org

772 lines, 445 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj)
9
10from core import error
11from core import num
12from core import state
13from display import pp_value
14from display import ui
15from core import vm
16from data_lang import j8
17from frontend import match
18from frontend import typed_args
19from mycpp import mops
20from mycpp import mylib
21from mycpp.mylib import NewDict, iteritems, log, tagswitch
22from ysh import expr_eval
23from ysh import val_ops
24
25from typing import TYPE_CHECKING, Dict, List, Optional, cast
26if TYPE_CHECKING:
27 from osh import glob_
28 from osh import split
29
30_ = log
31
32
33class Object(vm._Callable):
34 """Create a value.Obj
35
36 The order of params follows JavaScript's Object.create():
37 var obj = Object(prototype, props)
38 """
39
40 def __init__(self):
41 # type: () -> None
42 pass
43
44 def Call(self, rd):
45 # type: (typed_args.Reader) -> value_t
46
47 prototype = rd.PosValue()
48 props = rd.PosDict()
49 rd.Done()
50
51 chain = None # type: Optional[Obj]
52 UP_prototype = prototype
53 with tagswitch(prototype) as case:
54 if case(value_e.Null):
55 pass
56 elif case(value_e.Obj):
57 prototype = cast(Obj, UP_prototype)
58 chain = prototype
59 else:
60 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
61 rd.BlamePos())
62
63 # Opposite order
64 return Obj(props, chain)
65
66
67class Prototype(vm._Callable):
68 """Get an object's prototype."""
69
70 def __init__(self):
71 # type: () -> None
72 pass
73
74 def Call(self, rd):
75 # type: (typed_args.Reader) -> value_t
76
77 # TODO
78 return value.Null
79
80
81class Len(vm._Callable):
82
83 def __init__(self):
84 # type: () -> None
85 pass
86
87 def Call(self, rd):
88 # type: (typed_args.Reader) -> value_t
89
90 x = rd.PosValue()
91 rd.Done()
92
93 UP_x = x
94 with tagswitch(x) as case:
95 if case(value_e.List):
96 x = cast(value.List, UP_x)
97 return num.ToBig(len(x.items))
98
99 elif case(value_e.Dict):
100 x = cast(value.Dict, UP_x)
101 return num.ToBig(len(x.d))
102
103 elif case(value_e.Str):
104 x = cast(value.Str, UP_x)
105 return num.ToBig(len(x.s))
106
107 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
108 rd.BlamePos())
109
110
111class Type(vm._Callable):
112
113 def __init__(self):
114 # type: () -> None
115 pass
116
117 def Call(self, rd):
118 # type: (typed_args.Reader) -> value_t
119
120 val = rd.PosValue()
121 rd.Done()
122
123 return value.Str(ui.ValType(val))
124
125
126class Join(vm._Callable):
127 """Both free function join() and List->join() method."""
128
129 def __init__(self):
130 # type: () -> None
131 pass
132
133 def Call(self, rd):
134 # type: (typed_args.Reader) -> value_t
135
136 li = rd.PosList()
137 delim = rd.OptionalStr(default_='')
138 rd.Done()
139
140 strs = [] # type: List[str]
141 for i, el in enumerate(li):
142 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
143
144 return value.Str(delim.join(strs))
145
146
147class Maybe(vm._Callable):
148
149 def __init__(self):
150 # type: () -> None
151 pass
152
153 def Call(self, rd):
154 # type: (typed_args.Reader) -> value_t
155
156 val = rd.PosValue()
157 rd.Done()
158
159 if val == value.Null:
160 return value.List([])
161
162 s = val_ops.ToStr(
163 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
164 rd.LeftParenToken())
165 if len(s):
166 return value.List([val]) # use val to avoid needlessly copy
167
168 return value.List([])
169
170
171class Bool(vm._Callable):
172
173 def __init__(self):
174 # type: () -> None
175 pass
176
177 def Call(self, rd):
178 # type: (typed_args.Reader) -> value_t
179
180 val = rd.PosValue()
181 rd.Done()
182
183 return value.Bool(val_ops.ToBool(val))
184
185
186class Int(vm._Callable):
187
188 def __init__(self):
189 # type: () -> None
190 pass
191
192 def Call(self, rd):
193 # type: (typed_args.Reader) -> value_t
194
195 val = rd.PosValue()
196 rd.Done()
197
198 UP_val = val
199 with tagswitch(val) as case:
200 if case(value_e.Int):
201 return val
202
203 elif case(value_e.Bool):
204 val = cast(value.Bool, UP_val)
205 return value.Int(mops.FromBool(val.b))
206
207 elif case(value_e.Float):
208 val = cast(value.Float, UP_val)
209 ok, big_int = mops.FromFloat(val.f)
210 if ok:
211 return value.Int(big_int)
212 else:
213 raise error.Expr(
214 "Can't convert float %s to Int" %
215 pp_value.FloatString(val.f), rd.BlamePos())
216
217 elif case(value_e.Str):
218 val = cast(value.Str, UP_val)
219 if not match.LooksLikeInteger(val.s):
220 raise error.Expr("Can't convert %s to Int" % val.s,
221 rd.BlamePos())
222
223 return value.Int(mops.FromStr(val.s))
224
225 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
226 rd.BlamePos())
227
228
229class Float(vm._Callable):
230
231 def __init__(self):
232 # type: () -> None
233 pass
234
235 def Call(self, rd):
236 # type: (typed_args.Reader) -> value_t
237
238 val = rd.PosValue()
239 rd.Done()
240
241 UP_val = val
242 with tagswitch(val) as case:
243 if case(value_e.Int):
244 val = cast(value.Int, UP_val)
245 return value.Float(mops.ToFloat(val.i))
246
247 elif case(value_e.Float):
248 return val
249
250 elif case(value_e.Str):
251 val = cast(value.Str, UP_val)
252 if not match.LooksLikeFloat(val.s):
253 raise error.Expr('Cannot convert %s to Float' % val.s,
254 rd.BlamePos())
255
256 return value.Float(float(val.s))
257
258 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
259 rd.BlamePos())
260
261
262class Str_(vm._Callable):
263
264 def __init__(self):
265 # type: () -> None
266 pass
267
268 def Call(self, rd):
269 # type: (typed_args.Reader) -> value_t
270
271 val = rd.PosValue()
272 rd.Done()
273
274 # TODO: Should we call Stringify here? That would handle Eggex.
275
276 UP_val = val
277 with tagswitch(val) as case:
278 if case(value_e.Int):
279 val = cast(value.Int, UP_val)
280 return value.Str(mops.ToStr(val.i))
281
282 elif case(value_e.Float):
283 val = cast(value.Float, UP_val)
284 return value.Str(str(val.f))
285
286 elif case(value_e.Str):
287 return val
288
289 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
290 rd.BlamePos())
291
292
293class List_(vm._Callable):
294
295 def __init__(self):
296 # type: () -> None
297 pass
298
299 def Call(self, rd):
300 # type: (typed_args.Reader) -> value_t
301
302 val = rd.PosValue()
303 rd.Done()
304
305 l = [] # type: List[value_t]
306 it = None # type: val_ops.Iterator
307 UP_val = val
308 with tagswitch(val) as case:
309 if case(value_e.List):
310 val = cast(value.List, UP_val)
311 it = val_ops.ListIterator(val)
312
313 elif case(value_e.Dict):
314 val = cast(value.Dict, UP_val)
315 it = val_ops.DictIterator(val)
316
317 elif case(value_e.Range):
318 val = cast(value.Range, UP_val)
319 it = val_ops.RangeIterator(val)
320
321 else:
322 raise error.TypeErr(val,
323 'list() expected Dict, List, or Range',
324 rd.BlamePos())
325
326 assert it is not None
327 while True:
328 first = it.FirstValue()
329 if first is None:
330 break
331 l.append(first)
332 it.Next()
333
334 return value.List(l)
335
336
337class DictFunc(vm._Callable):
338
339 def __init__(self):
340 # type: () -> None
341 pass
342
343 def Call(self, rd):
344 # type: (typed_args.Reader) -> value_t
345
346 val = rd.PosValue()
347 rd.Done()
348
349 UP_val = val
350 with tagswitch(val) as case:
351 if case(value_e.Dict):
352 d = NewDict() # type: Dict[str, value_t]
353 val = cast(value.Dict, UP_val)
354 for k, v in iteritems(val.d):
355 d[k] = v
356
357 return value.Dict(d)
358
359 elif case(value_e.Obj):
360 d = NewDict()
361 val = cast(Obj, UP_val)
362 for k, v in iteritems(val.d):
363 d[k] = v
364
365 return value.Dict(d)
366
367 elif case(value_e.BashAssoc):
368 d = NewDict()
369 val = cast(value.BashAssoc, UP_val)
370 for k, s in iteritems(val.d):
371 d[k] = value.Str(s)
372
373 return value.Dict(d)
374
375 raise error.TypeErr(val, 'dict() expected Dict, Obj, or BashAssoc',
376 rd.BlamePos())
377
378
379class Runes(vm._Callable):
380
381 def __init__(self):
382 # type: () -> None
383 pass
384
385 def Call(self, rd):
386 # type: (typed_args.Reader) -> value_t
387 return value.Null
388
389
390class EncodeRunes(vm._Callable):
391
392 def __init__(self):
393 # type: () -> None
394 pass
395
396 def Call(self, rd):
397 # type: (typed_args.Reader) -> value_t
398 return value.Null
399
400
401class Bytes(vm._Callable):
402
403 def __init__(self):
404 # type: () -> None
405 pass
406
407 def Call(self, rd):
408 # type: (typed_args.Reader) -> value_t
409 return value.Null
410
411
412class EncodeBytes(vm._Callable):
413
414 def __init__(self):
415 # type: () -> None
416 pass
417
418 def Call(self, rd):
419 # type: (typed_args.Reader) -> value_t
420 return value.Null
421
422
423class Split(vm._Callable):
424
425 def __init__(self, splitter):
426 # type: (split.SplitContext) -> None
427 vm._Callable.__init__(self)
428 self.splitter = splitter
429
430 def Call(self, rd):
431 # type: (typed_args.Reader) -> value_t
432 s = rd.PosStr()
433
434 ifs = rd.OptionalStr()
435
436 rd.Done()
437
438 l = [
439 value.Str(elem)
440 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
441 ] # type: List[value_t]
442 return value.List(l)
443
444
445class FloatsEqual(vm._Callable):
446
447 def __init__(self):
448 # type: () -> None
449 pass
450
451 def Call(self, rd):
452 # type: (typed_args.Reader) -> value_t
453 left = rd.PosFloat()
454 right = rd.PosFloat()
455 rd.Done()
456
457 return value.Bool(left == right)
458
459
460class Glob(vm._Callable):
461
462 def __init__(self, globber):
463 # type: (glob_.Globber) -> None
464 vm._Callable.__init__(self)
465 self.globber = globber
466
467 def Call(self, rd):
468 # type: (typed_args.Reader) -> value_t
469 s = rd.PosStr()
470 rd.Done()
471
472 out = [] # type: List[str]
473 self.globber._Glob(s, out)
474
475 l = [value.Str(elem) for elem in out] # type: List[value_t]
476 return value.List(l)
477
478
479class Shvar_get(vm._Callable):
480 """Look up with dynamic scope."""
481
482 def __init__(self, mem):
483 # type: (state.Mem) -> None
484 vm._Callable.__init__(self)
485 self.mem = mem
486
487 def Call(self, rd):
488 # type: (typed_args.Reader) -> value_t
489 name = rd.PosStr()
490 rd.Done()
491 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
492
493
494class GetVar(vm._Callable):
495 """Look up normal scoping rules."""
496
497 def __init__(self, mem):
498 # type: (state.Mem) -> None
499 vm._Callable.__init__(self)
500 self.mem = mem
501
502 def Call(self, rd):
503 # type: (typed_args.Reader) -> value_t
504 name = rd.PosStr()
505 rd.Done()
506 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
507
508
509class EvalExpr(vm._Callable):
510
511 def __init__(self, expr_ev):
512 # type: (expr_eval.ExprEvaluator) -> None
513 self.expr_ev = expr_ev
514
515 def Call(self, rd):
516 # type: (typed_args.Reader) -> value_t
517 lazy = rd.PosExpr()
518 rd.Done()
519
520 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
521
522 return result
523
524
525class ToJson8(vm._Callable):
526
527 def __init__(self, is_j8):
528 # type: (bool) -> None
529 self.is_j8 = is_j8
530
531 def Call(self, rd):
532 # type: (typed_args.Reader) -> value_t
533
534 val = rd.PosValue()
535 space = mops.BigTruncate(rd.NamedInt('space', 0))
536 rd.Done()
537
538 # Convert from external JS-like API to internal API.
539 if space <= 0:
540 indent = -1
541 else:
542 indent = space
543
544 buf = mylib.BufWriter()
545 try:
546 if self.is_j8:
547 j8.PrintMessage(val, buf, indent)
548 else:
549 j8.PrintJsonMessage(val, buf, indent)
550 except error.Encode as e:
551 # status code 4 is special, for encode/decode errors.
552 raise error.Structured(4, e.Message(), rd.LeftParenToken())
553
554 return value.Str(buf.getvalue())
555
556
557class FromJson8(vm._Callable):
558
559 def __init__(self, is_j8):
560 # type: (bool) -> None
561 self.is_j8 = is_j8
562
563 def Call(self, rd):
564 # type: (typed_args.Reader) -> value_t
565
566 s = rd.PosStr()
567 rd.Done()
568
569 p = j8.Parser(s, self.is_j8)
570 try:
571 val = p.ParseValue()
572 except error.Decode as e:
573 # Right now I'm not exposing the original string, because that
574 # could lead to a memory leak in the _error Dict.
575 # The message quotes part of the string, and we could improve
576 # that. We could have a substring with context.
577 props = {
578 'start_pos': num.ToBig(e.start_pos),
579 'end_pos': num.ToBig(e.end_pos),
580 } # type: Dict[str, value_t]
581 # status code 4 is special, for encode/decode errors.
582 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
583
584 return val
585
586
587class BashArrayToSparse(vm._Callable):
588 """
589 value.BashArray -> value.SparseArray, for testing
590 """
591
592 def __init__(self):
593 # type: () -> None
594 pass
595
596 def Call(self, rd):
597 # type: (typed_args.Reader) -> value_t
598
599 strs = rd.PosBashArray()
600 rd.Done()
601
602 d = {} # type: Dict[mops.BigInt, str]
603 max_index = mops.MINUS_ONE # max index for empty array
604 for i, s in enumerate(strs):
605 if s is not None:
606 big_i = mops.IntWiden(i)
607 d[big_i] = s
608 if mops.Greater(big_i, max_index):
609 max_index = big_i
610
611 return value.SparseArray(d, max_index)
612
613
614class SparseOp(vm._Callable):
615 """
616 All ops on value.SparseArray, for testing performance
617 """
618
619 def __init__(self):
620 # type: () -> None
621 pass
622
623 def Call(self, rd):
624 # type: (typed_args.Reader) -> value_t
625
626 sp = rd.PosSparseArray()
627 d = sp.d
628 #i = mops.BigTruncate(rd.PosInt())
629 op_name = rd.PosStr()
630
631 no_str = None # type: str
632
633 if op_name == 'len': # ${#a[@]}
634 rd.Done()
635 return num.ToBig(len(d))
636
637 elif op_name == 'get': # ${a[42]}
638 index = rd.PosInt()
639 rd.Done()
640
641 s = d.get(index)
642 if s is None:
643 return value.Null
644 else:
645 return value.Str(s)
646
647 elif op_name == 'set': # a[42]=foo
648 index = rd.PosInt()
649 s = rd.PosStr()
650 rd.Done()
651
652 d[index] = s
653
654 if mops.Greater(index, sp.max_index):
655 sp.max_index = index
656
657 return value.Int(mops.ZERO)
658
659 elif op_name == 'unset': # unset 'a[1]'
660 index = rd.PosInt()
661 rd.Done()
662
663 mylib.dict_erase(d, index)
664
665 max_index = mops.MINUS_ONE # Note: this works if d is not empty
666 for i1 in d:
667 if mops.Greater(i1, max_index): # i1 > max_index
668 max_index = i1
669 sp.max_index = max_index
670
671 return value.Int(mops.ZERO)
672
673 elif op_name == 'subst': # "${a[@]}"
674 # Algorithm to expand a Dict[BigInt, Str]
675 #
676 # 1. Copy the integer keys into a new List
677 # 2. Sort them in numeric order
678 # 3. Create a List[str] that's the same size as the keys
679 # 4. Loop through sorted keys, look up value, and populate list
680 #
681 # There is another possible algorithm:
682 #
683 # 1. Copy the VALUES into a new list
684 # 2. Somehow sort them by the CORRESPONDING key, which depends on
685 # Slab<> POSITION. I think this does not fit within the
686 # std::sort() model. I think we would have to write a little custom
687 # sort algorithm.
688
689 keys = d.keys()
690 mylib.BigIntSort(keys)
691 # Pre-allocate
692 items = [no_str] * len(d) # type: List[str]
693 j = 0
694 for i in keys:
695 s = d.get(i)
696 assert s is not None
697 items[j] = s
698 j += 1
699 return value.BashArray(items)
700
701 elif op_name == 'keys': # "${!a[@]}"
702 keys = d.keys()
703 mylib.BigIntSort(keys)
704 items = [mops.ToStr(k) for k in keys]
705
706 # TODO: return SparseArray
707 return value.BashArray(items)
708
709 elif op_name == 'slice': # "${a[@]:0:5}"
710 start = rd.PosInt()
711 end = rd.PosInt()
712 rd.Done()
713
714 n = mops.BigTruncate(mops.Sub(end, start))
715 #log('start %d - end %d', start.i, end.i)
716
717 # Pre-allocate
718 items2 = [no_str] * n # type: List[str]
719
720 # Iterate from start to end. Note that this algorithm is
721 # theoretically slower than bash in the case where the array is
722 # sparse (in the part selected by the slice)
723 #
724 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
725 # elements, OSH will iterate through 999 integers and do 999 dict
726 # lookups, while bash will follow 3 pointers.
727 #
728 # However, in practice, I think iterating through integers is
729 # cheap.
730
731 j = 0
732 i = start
733 while mops.Greater(end, i): # i < end
734 s = d.get(i)
735 #log('s %s', s)
736 if s is not None:
737 items2[j] = s
738 j += 1
739
740 i = mops.Add(i, mops.ONE) # i += 1
741
742 # TODO: return SparseArray
743 return value.BashArray(items2)
744
745 elif op_name == 'append': # a+=(x y)
746 strs = rd.PosBashArray()
747
748 # TODO: We can maintain the max index in the value.SparseArray(),
749 # so that it's O(1) to append rather than O(n)
750 # - Update on 'set' is O(1)
751 # - Update on 'unset' is potentially O(n)
752
753 if 0:
754 max_index = mops.MINUS_ONE # Note: this works for empty arrays
755 for i1 in d:
756 if mops.Greater(i1, max_index): # i1 > max_index
757 max_index = i1
758 else:
759 max_index = sp.max_index
760
761 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
762 for s in strs:
763 d[i2] = s
764 i2 = mops.Add(i2, mops.ONE) # i2 += 1
765
766 # sp.max_index += len(strs)
767 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
768 return value.Int(mops.ZERO)
769
770 else:
771 print('Invalid SparseArray operation %r' % op_name)
772 return value.Int(mops.ZERO)