OILS / builtin / func_misc.py View on Github | oilshell.org

750 lines, 434 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj)
9
10from core import error
11from core import num
12from core import state
13from display import pp_value
14from display import ui
15from core import vm
16from data_lang import j8
17from frontend import match
18from frontend import typed_args
19from mycpp import mops
20from mycpp import mylib
21from mycpp.mylib import NewDict, iteritems, log, tagswitch
22from ysh import expr_eval
23from ysh import val_ops
24
25from typing import TYPE_CHECKING, Dict, List, Optional, cast
26if TYPE_CHECKING:
27 from osh import glob_
28 from osh import split
29
30_ = log
31
32
33class Object(vm._Callable):
34 """
35 Create an object. The order of params follows JavaScript's Object.create()
36
37 var obj = Object(prototype, props)
38 """
39
40 def __init__(self):
41 # type: () -> None
42 pass
43
44 def Call(self, rd):
45 # type: (typed_args.Reader) -> value_t
46
47 prototype = rd.PosValue()
48 props = rd.PosDict()
49 rd.Done()
50
51 chain = None # type: Optional[Obj]
52 UP_prototype = prototype
53 with tagswitch(prototype) as case:
54 if case(value_e.Null):
55 pass
56 elif case(value_e.Obj):
57 prototype = cast(Obj, UP_prototype)
58 chain = prototype
59 else:
60 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
61 rd.BlamePos())
62
63 # Opposite order
64 return Obj(props, chain)
65
66
67class Len(vm._Callable):
68
69 def __init__(self):
70 # type: () -> None
71 pass
72
73 def Call(self, rd):
74 # type: (typed_args.Reader) -> value_t
75
76 x = rd.PosValue()
77 rd.Done()
78
79 UP_x = x
80 with tagswitch(x) as case:
81 if case(value_e.List):
82 x = cast(value.List, UP_x)
83 return num.ToBig(len(x.items))
84
85 elif case(value_e.Dict):
86 x = cast(value.Dict, UP_x)
87 return num.ToBig(len(x.d))
88
89 elif case(value_e.Str):
90 x = cast(value.Str, UP_x)
91 return num.ToBig(len(x.s))
92
93 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
94 rd.BlamePos())
95
96
97class Type(vm._Callable):
98
99 def __init__(self):
100 # type: () -> None
101 pass
102
103 def Call(self, rd):
104 # type: (typed_args.Reader) -> value_t
105
106 val = rd.PosValue()
107 rd.Done()
108
109 return value.Str(ui.ValType(val))
110
111
112class Join(vm._Callable):
113 """Both free function join() and List->join() method."""
114
115 def __init__(self):
116 # type: () -> None
117 pass
118
119 def Call(self, rd):
120 # type: (typed_args.Reader) -> value_t
121
122 li = rd.PosList()
123 delim = rd.OptionalStr(default_='')
124 rd.Done()
125
126 strs = [] # type: List[str]
127 for i, el in enumerate(li):
128 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
129
130 return value.Str(delim.join(strs))
131
132
133class Maybe(vm._Callable):
134
135 def __init__(self):
136 # type: () -> None
137 pass
138
139 def Call(self, rd):
140 # type: (typed_args.Reader) -> value_t
141
142 val = rd.PosValue()
143 rd.Done()
144
145 if val == value.Null:
146 return value.List([])
147
148 s = val_ops.ToStr(
149 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
150 rd.LeftParenToken())
151 if len(s):
152 return value.List([val]) # use val to avoid needlessly copy
153
154 return value.List([])
155
156
157class Bool(vm._Callable):
158
159 def __init__(self):
160 # type: () -> None
161 pass
162
163 def Call(self, rd):
164 # type: (typed_args.Reader) -> value_t
165
166 val = rd.PosValue()
167 rd.Done()
168
169 return value.Bool(val_ops.ToBool(val))
170
171
172class Int(vm._Callable):
173
174 def __init__(self):
175 # type: () -> None
176 pass
177
178 def Call(self, rd):
179 # type: (typed_args.Reader) -> value_t
180
181 val = rd.PosValue()
182 rd.Done()
183
184 UP_val = val
185 with tagswitch(val) as case:
186 if case(value_e.Int):
187 return val
188
189 elif case(value_e.Bool):
190 val = cast(value.Bool, UP_val)
191 return value.Int(mops.FromBool(val.b))
192
193 elif case(value_e.Float):
194 val = cast(value.Float, UP_val)
195 ok, big_int = mops.FromFloat(val.f)
196 if ok:
197 return value.Int(big_int)
198 else:
199 raise error.Expr(
200 "Can't convert float %s to Int" %
201 pp_value.FloatString(val.f), rd.BlamePos())
202
203 elif case(value_e.Str):
204 val = cast(value.Str, UP_val)
205 if not match.LooksLikeInteger(val.s):
206 raise error.Expr("Can't convert %s to Int" % val.s,
207 rd.BlamePos())
208
209 return value.Int(mops.FromStr(val.s))
210
211 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
212 rd.BlamePos())
213
214
215class Float(vm._Callable):
216
217 def __init__(self):
218 # type: () -> None
219 pass
220
221 def Call(self, rd):
222 # type: (typed_args.Reader) -> value_t
223
224 val = rd.PosValue()
225 rd.Done()
226
227 UP_val = val
228 with tagswitch(val) as case:
229 if case(value_e.Int):
230 val = cast(value.Int, UP_val)
231 return value.Float(mops.ToFloat(val.i))
232
233 elif case(value_e.Float):
234 return val
235
236 elif case(value_e.Str):
237 val = cast(value.Str, UP_val)
238 if not match.LooksLikeFloat(val.s):
239 raise error.Expr('Cannot convert %s to Float' % val.s,
240 rd.BlamePos())
241
242 return value.Float(float(val.s))
243
244 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
245 rd.BlamePos())
246
247
248class Str_(vm._Callable):
249
250 def __init__(self):
251 # type: () -> None
252 pass
253
254 def Call(self, rd):
255 # type: (typed_args.Reader) -> value_t
256
257 val = rd.PosValue()
258 rd.Done()
259
260 # TODO: Should we call Stringify here? That would handle Eggex.
261
262 UP_val = val
263 with tagswitch(val) as case:
264 if case(value_e.Int):
265 val = cast(value.Int, UP_val)
266 return value.Str(mops.ToStr(val.i))
267
268 elif case(value_e.Float):
269 val = cast(value.Float, UP_val)
270 return value.Str(str(val.f))
271
272 elif case(value_e.Str):
273 return val
274
275 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
276 rd.BlamePos())
277
278
279class List_(vm._Callable):
280
281 def __init__(self):
282 # type: () -> None
283 pass
284
285 def Call(self, rd):
286 # type: (typed_args.Reader) -> value_t
287
288 val = rd.PosValue()
289 rd.Done()
290
291 l = [] # type: List[value_t]
292 it = None # type: val_ops.Iterator
293 UP_val = val
294 with tagswitch(val) as case:
295 if case(value_e.List):
296 val = cast(value.List, UP_val)
297 it = val_ops.ListIterator(val)
298
299 elif case(value_e.Dict):
300 val = cast(value.Dict, UP_val)
301 it = val_ops.DictIterator(val)
302
303 elif case(value_e.Range):
304 val = cast(value.Range, UP_val)
305 it = val_ops.RangeIterator(val)
306
307 else:
308 raise error.TypeErr(val,
309 'list() expected Dict, List, or Range',
310 rd.BlamePos())
311
312 assert it is not None
313 while True:
314 first = it.FirstValue()
315 if first is None:
316 break
317 l.append(first)
318 it.Next()
319
320 return value.List(l)
321
322
323class DictFunc(vm._Callable):
324
325 def __init__(self):
326 # type: () -> None
327 pass
328
329 def Call(self, rd):
330 # type: (typed_args.Reader) -> value_t
331
332 val = rd.PosValue()
333 rd.Done()
334
335 UP_val = val
336 with tagswitch(val) as case:
337 if case(value_e.Dict):
338 d = NewDict() # type: Dict[str, value_t]
339 val = cast(value.Dict, UP_val)
340 for k, v in iteritems(val.d):
341 d[k] = v
342
343 return value.Dict(d)
344
345 elif case(value_e.BashAssoc):
346 d = NewDict()
347 val = cast(value.BashAssoc, UP_val)
348 for k, s in iteritems(val.d):
349 d[k] = value.Str(s)
350
351 return value.Dict(d)
352
353 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
354 rd.BlamePos())
355
356
357class Runes(vm._Callable):
358
359 def __init__(self):
360 # type: () -> None
361 pass
362
363 def Call(self, rd):
364 # type: (typed_args.Reader) -> value_t
365 return value.Null
366
367
368class EncodeRunes(vm._Callable):
369
370 def __init__(self):
371 # type: () -> None
372 pass
373
374 def Call(self, rd):
375 # type: (typed_args.Reader) -> value_t
376 return value.Null
377
378
379class Bytes(vm._Callable):
380
381 def __init__(self):
382 # type: () -> None
383 pass
384
385 def Call(self, rd):
386 # type: (typed_args.Reader) -> value_t
387 return value.Null
388
389
390class EncodeBytes(vm._Callable):
391
392 def __init__(self):
393 # type: () -> None
394 pass
395
396 def Call(self, rd):
397 # type: (typed_args.Reader) -> value_t
398 return value.Null
399
400
401class Split(vm._Callable):
402
403 def __init__(self, splitter):
404 # type: (split.SplitContext) -> None
405 vm._Callable.__init__(self)
406 self.splitter = splitter
407
408 def Call(self, rd):
409 # type: (typed_args.Reader) -> value_t
410 s = rd.PosStr()
411
412 ifs = rd.OptionalStr()
413
414 rd.Done()
415
416 l = [
417 value.Str(elem)
418 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
419 ] # type: List[value_t]
420 return value.List(l)
421
422
423class FloatsEqual(vm._Callable):
424
425 def __init__(self):
426 # type: () -> None
427 pass
428
429 def Call(self, rd):
430 # type: (typed_args.Reader) -> value_t
431 left = rd.PosFloat()
432 right = rd.PosFloat()
433 rd.Done()
434
435 return value.Bool(left == right)
436
437
438class Glob(vm._Callable):
439
440 def __init__(self, globber):
441 # type: (glob_.Globber) -> None
442 vm._Callable.__init__(self)
443 self.globber = globber
444
445 def Call(self, rd):
446 # type: (typed_args.Reader) -> value_t
447 s = rd.PosStr()
448 rd.Done()
449
450 out = [] # type: List[str]
451 self.globber._Glob(s, out)
452
453 l = [value.Str(elem) for elem in out] # type: List[value_t]
454 return value.List(l)
455
456
457class Shvar_get(vm._Callable):
458 """Look up with dynamic scope."""
459
460 def __init__(self, mem):
461 # type: (state.Mem) -> None
462 vm._Callable.__init__(self)
463 self.mem = mem
464
465 def Call(self, rd):
466 # type: (typed_args.Reader) -> value_t
467 name = rd.PosStr()
468 rd.Done()
469 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
470
471
472class GetVar(vm._Callable):
473 """Look up normal scoping rules."""
474
475 def __init__(self, mem):
476 # type: (state.Mem) -> None
477 vm._Callable.__init__(self)
478 self.mem = mem
479
480 def Call(self, rd):
481 # type: (typed_args.Reader) -> value_t
482 name = rd.PosStr()
483 rd.Done()
484 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
485
486
487class EvalExpr(vm._Callable):
488
489 def __init__(self, expr_ev):
490 # type: (expr_eval.ExprEvaluator) -> None
491 self.expr_ev = expr_ev
492
493 def Call(self, rd):
494 # type: (typed_args.Reader) -> value_t
495 lazy = rd.PosExpr()
496 rd.Done()
497
498 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
499
500 return result
501
502
503class ToJson8(vm._Callable):
504
505 def __init__(self, is_j8):
506 # type: (bool) -> None
507 self.is_j8 = is_j8
508
509 def Call(self, rd):
510 # type: (typed_args.Reader) -> value_t
511
512 val = rd.PosValue()
513 space = mops.BigTruncate(rd.NamedInt('space', 0))
514 rd.Done()
515
516 # Convert from external JS-like API to internal API.
517 if space <= 0:
518 indent = -1
519 else:
520 indent = space
521
522 buf = mylib.BufWriter()
523 try:
524 if self.is_j8:
525 j8.PrintMessage(val, buf, indent)
526 else:
527 j8.PrintJsonMessage(val, buf, indent)
528 except error.Encode as e:
529 # status code 4 is special, for encode/decode errors.
530 raise error.Structured(4, e.Message(), rd.LeftParenToken())
531
532 return value.Str(buf.getvalue())
533
534
535class FromJson8(vm._Callable):
536
537 def __init__(self, is_j8):
538 # type: (bool) -> None
539 self.is_j8 = is_j8
540
541 def Call(self, rd):
542 # type: (typed_args.Reader) -> value_t
543
544 s = rd.PosStr()
545 rd.Done()
546
547 p = j8.Parser(s, self.is_j8)
548 try:
549 val = p.ParseValue()
550 except error.Decode as e:
551 # Right now I'm not exposing the original string, because that
552 # could lead to a memory leak in the _error Dict.
553 # The message quotes part of the string, and we could improve
554 # that. We could have a substring with context.
555 props = {
556 'start_pos': num.ToBig(e.start_pos),
557 'end_pos': num.ToBig(e.end_pos),
558 } # type: Dict[str, value_t]
559 # status code 4 is special, for encode/decode errors.
560 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
561
562 return val
563
564
565class BashArrayToSparse(vm._Callable):
566 """
567 value.BashArray -> value.SparseArray, for testing
568 """
569
570 def __init__(self):
571 # type: () -> None
572 pass
573
574 def Call(self, rd):
575 # type: (typed_args.Reader) -> value_t
576
577 strs = rd.PosBashArray()
578 rd.Done()
579
580 d = {} # type: Dict[mops.BigInt, str]
581 max_index = mops.MINUS_ONE # max index for empty array
582 for i, s in enumerate(strs):
583 if s is not None:
584 big_i = mops.IntWiden(i)
585 d[big_i] = s
586 if mops.Greater(big_i, max_index):
587 max_index = big_i
588
589 return value.SparseArray(d, max_index)
590
591
592class SparseOp(vm._Callable):
593 """
594 All ops on value.SparseArray, for testing performance
595 """
596
597 def __init__(self):
598 # type: () -> None
599 pass
600
601 def Call(self, rd):
602 # type: (typed_args.Reader) -> value_t
603
604 sp = rd.PosSparseArray()
605 d = sp.d
606 #i = mops.BigTruncate(rd.PosInt())
607 op_name = rd.PosStr()
608
609 no_str = None # type: str
610
611 if op_name == 'len': # ${#a[@]}
612 rd.Done()
613 return num.ToBig(len(d))
614
615 elif op_name == 'get': # ${a[42]}
616 index = rd.PosInt()
617 rd.Done()
618
619 s = d.get(index)
620 if s is None:
621 return value.Null
622 else:
623 return value.Str(s)
624
625 elif op_name == 'set': # a[42]=foo
626 index = rd.PosInt()
627 s = rd.PosStr()
628 rd.Done()
629
630 d[index] = s
631
632 if mops.Greater(index, sp.max_index):
633 sp.max_index = index
634
635 return value.Int(mops.ZERO)
636
637 elif op_name == 'unset': # unset 'a[1]'
638 index = rd.PosInt()
639 rd.Done()
640
641 mylib.dict_erase(d, index)
642
643 max_index = mops.MINUS_ONE # Note: this works if d is not empty
644 for i1 in d:
645 if mops.Greater(i1, max_index): # i1 > max_index
646 max_index = i1
647 sp.max_index = max_index
648
649 return value.Int(mops.ZERO)
650
651 elif op_name == 'subst': # "${a[@]}"
652 # Algorithm to expand a Dict[BigInt, Str]
653 #
654 # 1. Copy the integer keys into a new List
655 # 2. Sort them in numeric order
656 # 3. Create a List[str] that's the same size as the keys
657 # 4. Loop through sorted keys, look up value, and populate list
658 #
659 # There is another possible algorithm:
660 #
661 # 1. Copy the VALUES into a new list
662 # 2. Somehow sort them by the CORRESPONDING key, which depends on
663 # Slab<> POSITION. I think this does not fit within the
664 # std::sort() model. I think we would have to write a little custom
665 # sort algorithm.
666
667 keys = d.keys()
668 mylib.BigIntSort(keys)
669 # Pre-allocate
670 items = [no_str] * len(d) # type: List[str]
671 j = 0
672 for i in keys:
673 s = d.get(i)
674 assert s is not None
675 items[j] = s
676 j += 1
677 return value.BashArray(items)
678
679 elif op_name == 'keys': # "${!a[@]}"
680 keys = d.keys()
681 mylib.BigIntSort(keys)
682 items = [mops.ToStr(k) for k in keys]
683
684 # TODO: return SparseArray
685 return value.BashArray(items)
686
687 elif op_name == 'slice': # "${a[@]:0:5}"
688 start = rd.PosInt()
689 end = rd.PosInt()
690 rd.Done()
691
692 n = mops.BigTruncate(mops.Sub(end, start))
693 #log('start %d - end %d', start.i, end.i)
694
695 # Pre-allocate
696 items2 = [no_str] * n # type: List[str]
697
698 # Iterate from start to end. Note that this algorithm is
699 # theoretically slower than bash in the case where the array is
700 # sparse (in the part selected by the slice)
701 #
702 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
703 # elements, OSH will iterate through 999 integers and do 999 dict
704 # lookups, while bash will follow 3 pointers.
705 #
706 # However, in practice, I think iterating through integers is
707 # cheap.
708
709 j = 0
710 i = start
711 while mops.Greater(end, i): # i < end
712 s = d.get(i)
713 #log('s %s', s)
714 if s is not None:
715 items2[j] = s
716 j += 1
717
718 i = mops.Add(i, mops.ONE) # i += 1
719
720 # TODO: return SparseArray
721 return value.BashArray(items2)
722
723 elif op_name == 'append': # a+=(x y)
724 strs = rd.PosBashArray()
725
726 # TODO: We can maintain the max index in the value.SparseArray(),
727 # so that it's O(1) to append rather than O(n)
728 # - Update on 'set' is O(1)
729 # - Update on 'unset' is potentially O(n)
730
731 if 0:
732 max_index = mops.MINUS_ONE # Note: this works for empty arrays
733 for i1 in d:
734 if mops.Greater(i1, max_index): # i1 > max_index
735 max_index = i1
736 else:
737 max_index = sp.max_index
738
739 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
740 for s in strs:
741 d[i2] = s
742 i2 = mops.Add(i2, mops.ONE) # i2 += 1
743
744 # sp.max_index += len(strs)
745 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
746 return value.Int(mops.ZERO)
747
748 else:
749 print('Invalid SparseArray operation %r' % op_name)
750 return value.Int(mops.ZERO)