OILS / builtin / func_misc.py View on Github | oilshell.org

739 lines, 424 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str,
9 Dict_)
10
11from core import error
12from core import num
13from core import state
14from display import pp_value
15from display import ui
16from core import vm
17from data_lang import j8
18from frontend import match
19from frontend import typed_args
20from mycpp import mops
21from mycpp import mylib
22from mycpp.mylib import NewDict, iteritems, log, tagswitch
23from ysh import expr_eval
24from ysh import val_ops
25
26from typing import TYPE_CHECKING, Dict, List, cast
27if TYPE_CHECKING:
28 from osh import glob_
29 from osh import split
30
31_ = log
32
33
34class Object(vm._Callable):
35 """
36 Create an object. The order of params follows JavaScript's Object.create()
37
38 var obj = Object(prototype, props)
39 """
40
41 def __init__(self):
42 # type: () -> None
43 pass
44
45 def Call(self, rd):
46 # type: (typed_args.Reader) -> value_t
47
48 prototype = rd.PosObject()
49 props = rd.PosDict()
50 rd.Done()
51
52 # Opposite order
53 return Dict_(props, prototype)
54
55
56class Len(vm._Callable):
57
58 def __init__(self):
59 # type: () -> None
60 pass
61
62 def Call(self, rd):
63 # type: (typed_args.Reader) -> value_t
64
65 x = rd.PosValue()
66 rd.Done()
67
68 UP_x = x
69 with tagswitch(x) as case:
70 if case(value_e.List):
71 x = cast(value.List, UP_x)
72 return num.ToBig(len(x.items))
73
74 elif case(value_e.Dict):
75 x = cast(Dict_, UP_x)
76 return num.ToBig(len(x.d))
77
78 elif case(value_e.Str):
79 x = cast(value.Str, UP_x)
80 return num.ToBig(len(x.s))
81
82 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
83 rd.BlamePos())
84
85
86class Type(vm._Callable):
87
88 def __init__(self):
89 # type: () -> None
90 pass
91
92 def Call(self, rd):
93 # type: (typed_args.Reader) -> value_t
94
95 val = rd.PosValue()
96 rd.Done()
97
98 return value.Str(ui.ValType(val))
99
100
101class Join(vm._Callable):
102 """Both free function join() and List->join() method."""
103
104 def __init__(self):
105 # type: () -> None
106 pass
107
108 def Call(self, rd):
109 # type: (typed_args.Reader) -> value_t
110
111 li = rd.PosList()
112 delim = rd.OptionalStr(default_='')
113 rd.Done()
114
115 strs = [] # type: List[str]
116 for i, el in enumerate(li):
117 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
118
119 return value.Str(delim.join(strs))
120
121
122class Maybe(vm._Callable):
123
124 def __init__(self):
125 # type: () -> None
126 pass
127
128 def Call(self, rd):
129 # type: (typed_args.Reader) -> value_t
130
131 val = rd.PosValue()
132 rd.Done()
133
134 if val == value.Null:
135 return value.List([])
136
137 s = val_ops.ToStr(
138 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
139 rd.LeftParenToken())
140 if len(s):
141 return value.List([val]) # use val to avoid needlessly copy
142
143 return value.List([])
144
145
146class Bool(vm._Callable):
147
148 def __init__(self):
149 # type: () -> None
150 pass
151
152 def Call(self, rd):
153 # type: (typed_args.Reader) -> value_t
154
155 val = rd.PosValue()
156 rd.Done()
157
158 return value.Bool(val_ops.ToBool(val))
159
160
161class Int(vm._Callable):
162
163 def __init__(self):
164 # type: () -> None
165 pass
166
167 def Call(self, rd):
168 # type: (typed_args.Reader) -> value_t
169
170 val = rd.PosValue()
171 rd.Done()
172
173 UP_val = val
174 with tagswitch(val) as case:
175 if case(value_e.Int):
176 return val
177
178 elif case(value_e.Bool):
179 val = cast(value.Bool, UP_val)
180 return value.Int(mops.FromBool(val.b))
181
182 elif case(value_e.Float):
183 val = cast(value.Float, UP_val)
184 ok, big_int = mops.FromFloat(val.f)
185 if ok:
186 return value.Int(big_int)
187 else:
188 raise error.Expr(
189 "Can't convert float %s to Int" %
190 pp_value.FloatString(val.f), rd.BlamePos())
191
192 elif case(value_e.Str):
193 val = cast(value.Str, UP_val)
194 if not match.LooksLikeInteger(val.s):
195 raise error.Expr("Can't convert %s to Int" % val.s,
196 rd.BlamePos())
197
198 return value.Int(mops.FromStr(val.s))
199
200 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
201 rd.BlamePos())
202
203
204class Float(vm._Callable):
205
206 def __init__(self):
207 # type: () -> None
208 pass
209
210 def Call(self, rd):
211 # type: (typed_args.Reader) -> value_t
212
213 val = rd.PosValue()
214 rd.Done()
215
216 UP_val = val
217 with tagswitch(val) as case:
218 if case(value_e.Int):
219 val = cast(value.Int, UP_val)
220 return value.Float(mops.ToFloat(val.i))
221
222 elif case(value_e.Float):
223 return val
224
225 elif case(value_e.Str):
226 val = cast(value.Str, UP_val)
227 if not match.LooksLikeFloat(val.s):
228 raise error.Expr('Cannot convert %s to Float' % val.s,
229 rd.BlamePos())
230
231 return value.Float(float(val.s))
232
233 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
234 rd.BlamePos())
235
236
237class Str_(vm._Callable):
238
239 def __init__(self):
240 # type: () -> None
241 pass
242
243 def Call(self, rd):
244 # type: (typed_args.Reader) -> value_t
245
246 val = rd.PosValue()
247 rd.Done()
248
249 # TODO: Should we call Stringify here? That would handle Eggex.
250
251 UP_val = val
252 with tagswitch(val) as case:
253 if case(value_e.Int):
254 val = cast(value.Int, UP_val)
255 return value.Str(mops.ToStr(val.i))
256
257 elif case(value_e.Float):
258 val = cast(value.Float, UP_val)
259 return value.Str(str(val.f))
260
261 elif case(value_e.Str):
262 return val
263
264 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
265 rd.BlamePos())
266
267
268class List_(vm._Callable):
269
270 def __init__(self):
271 # type: () -> None
272 pass
273
274 def Call(self, rd):
275 # type: (typed_args.Reader) -> value_t
276
277 val = rd.PosValue()
278 rd.Done()
279
280 l = [] # type: List[value_t]
281 it = None # type: val_ops.Iterator
282 UP_val = val
283 with tagswitch(val) as case:
284 if case(value_e.List):
285 val = cast(value.List, UP_val)
286 it = val_ops.ListIterator(val)
287
288 elif case(value_e.Dict):
289 val = cast(Dict_, UP_val)
290 it = val_ops.DictIterator(val)
291
292 elif case(value_e.Range):
293 val = cast(value.Range, UP_val)
294 it = val_ops.RangeIterator(val)
295
296 else:
297 raise error.TypeErr(val,
298 'list() expected Dict, List, or Range',
299 rd.BlamePos())
300
301 assert it is not None
302 while True:
303 first = it.FirstValue()
304 if first is None:
305 break
306 l.append(first)
307 it.Next()
308
309 return value.List(l)
310
311
312class DictFunc(vm._Callable):
313
314 def __init__(self):
315 # type: () -> None
316 pass
317
318 def Call(self, rd):
319 # type: (typed_args.Reader) -> value_t
320
321 val = rd.PosValue()
322 rd.Done()
323
324 UP_val = val
325 with tagswitch(val) as case:
326 if case(value_e.Dict):
327 d = NewDict() # type: Dict[str, value_t]
328 val = cast(Dict_, UP_val)
329 for k, v in iteritems(val.d):
330 d[k] = v
331
332 return Dict_(d, None)
333
334 elif case(value_e.BashAssoc):
335 d = NewDict()
336 val = cast(value.BashAssoc, UP_val)
337 for k, s in iteritems(val.d):
338 d[k] = value.Str(s)
339
340 return Dict_(d, None)
341
342 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
343 rd.BlamePos())
344
345
346class Runes(vm._Callable):
347
348 def __init__(self):
349 # type: () -> None
350 pass
351
352 def Call(self, rd):
353 # type: (typed_args.Reader) -> value_t
354 return value.Null
355
356
357class EncodeRunes(vm._Callable):
358
359 def __init__(self):
360 # type: () -> None
361 pass
362
363 def Call(self, rd):
364 # type: (typed_args.Reader) -> value_t
365 return value.Null
366
367
368class Bytes(vm._Callable):
369
370 def __init__(self):
371 # type: () -> None
372 pass
373
374 def Call(self, rd):
375 # type: (typed_args.Reader) -> value_t
376 return value.Null
377
378
379class EncodeBytes(vm._Callable):
380
381 def __init__(self):
382 # type: () -> None
383 pass
384
385 def Call(self, rd):
386 # type: (typed_args.Reader) -> value_t
387 return value.Null
388
389
390class Split(vm._Callable):
391
392 def __init__(self, splitter):
393 # type: (split.SplitContext) -> None
394 vm._Callable.__init__(self)
395 self.splitter = splitter
396
397 def Call(self, rd):
398 # type: (typed_args.Reader) -> value_t
399 s = rd.PosStr()
400
401 ifs = rd.OptionalStr()
402
403 rd.Done()
404
405 l = [
406 value.Str(elem)
407 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
408 ] # type: List[value_t]
409 return value.List(l)
410
411
412class FloatsEqual(vm._Callable):
413
414 def __init__(self):
415 # type: () -> None
416 pass
417
418 def Call(self, rd):
419 # type: (typed_args.Reader) -> value_t
420 left = rd.PosFloat()
421 right = rd.PosFloat()
422 rd.Done()
423
424 return value.Bool(left == right)
425
426
427class Glob(vm._Callable):
428
429 def __init__(self, globber):
430 # type: (glob_.Globber) -> None
431 vm._Callable.__init__(self)
432 self.globber = globber
433
434 def Call(self, rd):
435 # type: (typed_args.Reader) -> value_t
436 s = rd.PosStr()
437 rd.Done()
438
439 out = [] # type: List[str]
440 self.globber._Glob(s, out)
441
442 l = [value.Str(elem) for elem in out] # type: List[value_t]
443 return value.List(l)
444
445
446class Shvar_get(vm._Callable):
447 """Look up with dynamic scope."""
448
449 def __init__(self, mem):
450 # type: (state.Mem) -> None
451 vm._Callable.__init__(self)
452 self.mem = mem
453
454 def Call(self, rd):
455 # type: (typed_args.Reader) -> value_t
456 name = rd.PosStr()
457 rd.Done()
458 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
459
460
461class GetVar(vm._Callable):
462 """Look up normal scoping rules."""
463
464 def __init__(self, mem):
465 # type: (state.Mem) -> None
466 vm._Callable.__init__(self)
467 self.mem = mem
468
469 def Call(self, rd):
470 # type: (typed_args.Reader) -> value_t
471 name = rd.PosStr()
472 rd.Done()
473 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
474
475
476class EvalExpr(vm._Callable):
477
478 def __init__(self, expr_ev):
479 # type: (expr_eval.ExprEvaluator) -> None
480 self.expr_ev = expr_ev
481
482 def Call(self, rd):
483 # type: (typed_args.Reader) -> value_t
484 lazy = rd.PosExpr()
485 rd.Done()
486
487 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
488
489 return result
490
491
492class ToJson8(vm._Callable):
493
494 def __init__(self, is_j8):
495 # type: (bool) -> None
496 self.is_j8 = is_j8
497
498 def Call(self, rd):
499 # type: (typed_args.Reader) -> value_t
500
501 val = rd.PosValue()
502 space = mops.BigTruncate(rd.NamedInt('space', 0))
503 rd.Done()
504
505 # Convert from external JS-like API to internal API.
506 if space <= 0:
507 indent = -1
508 else:
509 indent = space
510
511 buf = mylib.BufWriter()
512 try:
513 if self.is_j8:
514 j8.PrintMessage(val, buf, indent)
515 else:
516 j8.PrintJsonMessage(val, buf, indent)
517 except error.Encode as e:
518 # status code 4 is special, for encode/decode errors.
519 raise error.Structured(4, e.Message(), rd.LeftParenToken())
520
521 return value.Str(buf.getvalue())
522
523
524class FromJson8(vm._Callable):
525
526 def __init__(self, is_j8):
527 # type: (bool) -> None
528 self.is_j8 = is_j8
529
530 def Call(self, rd):
531 # type: (typed_args.Reader) -> value_t
532
533 s = rd.PosStr()
534 rd.Done()
535
536 p = j8.Parser(s, self.is_j8)
537 try:
538 val = p.ParseValue()
539 except error.Decode as e:
540 # Right now I'm not exposing the original string, because that
541 # could lead to a memory leak in the _error Dict.
542 # The message quotes part of the string, and we could improve
543 # that. We could have a substring with context.
544 props = {
545 'start_pos': num.ToBig(e.start_pos),
546 'end_pos': num.ToBig(e.end_pos),
547 } # type: Dict[str, value_t]
548 # status code 4 is special, for encode/decode errors.
549 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
550
551 return val
552
553
554class BashArrayToSparse(vm._Callable):
555 """
556 value.BashArray -> value.SparseArray, for testing
557 """
558
559 def __init__(self):
560 # type: () -> None
561 pass
562
563 def Call(self, rd):
564 # type: (typed_args.Reader) -> value_t
565
566 strs = rd.PosBashArray()
567 rd.Done()
568
569 d = {} # type: Dict[mops.BigInt, str]
570 max_index = mops.MINUS_ONE # max index for empty array
571 for i, s in enumerate(strs):
572 if s is not None:
573 big_i = mops.IntWiden(i)
574 d[big_i] = s
575 if mops.Greater(big_i, max_index):
576 max_index = big_i
577
578 return value.SparseArray(d, max_index)
579
580
581class SparseOp(vm._Callable):
582 """
583 All ops on value.SparseArray, for testing performance
584 """
585
586 def __init__(self):
587 # type: () -> None
588 pass
589
590 def Call(self, rd):
591 # type: (typed_args.Reader) -> value_t
592
593 sp = rd.PosSparseArray()
594 d = sp.d
595 #i = mops.BigTruncate(rd.PosInt())
596 op_name = rd.PosStr()
597
598 no_str = None # type: str
599
600 if op_name == 'len': # ${#a[@]}
601 rd.Done()
602 return num.ToBig(len(d))
603
604 elif op_name == 'get': # ${a[42]}
605 index = rd.PosInt()
606 rd.Done()
607
608 s = d.get(index)
609 if s is None:
610 return value.Null
611 else:
612 return value.Str(s)
613
614 elif op_name == 'set': # a[42]=foo
615 index = rd.PosInt()
616 s = rd.PosStr()
617 rd.Done()
618
619 d[index] = s
620
621 if mops.Greater(index, sp.max_index):
622 sp.max_index = index
623
624 return value.Int(mops.ZERO)
625
626 elif op_name == 'unset': # unset 'a[1]'
627 index = rd.PosInt()
628 rd.Done()
629
630 mylib.dict_erase(d, index)
631
632 max_index = mops.MINUS_ONE # Note: this works if d is not empty
633 for i1 in d:
634 if mops.Greater(i1, max_index): # i1 > max_index
635 max_index = i1
636 sp.max_index = max_index
637
638 return value.Int(mops.ZERO)
639
640 elif op_name == 'subst': # "${a[@]}"
641 # Algorithm to expand a Dict[BigInt, Str]
642 #
643 # 1. Copy the integer keys into a new List
644 # 2. Sort them in numeric order
645 # 3. Create a List[str] that's the same size as the keys
646 # 4. Loop through sorted keys, look up value, and populate list
647 #
648 # There is another possible algorithm:
649 #
650 # 1. Copy the VALUES into a new list
651 # 2. Somehow sort them by the CORRESPONDING key, which depends on
652 # Slab<> POSITION. I think this does not fit within the
653 # std::sort() model. I think we would have to write a little custom
654 # sort algorithm.
655
656 keys = d.keys()
657 mylib.BigIntSort(keys)
658 # Pre-allocate
659 items = [no_str] * len(d) # type: List[str]
660 j = 0
661 for i in keys:
662 s = d.get(i)
663 assert s is not None
664 items[j] = s
665 j += 1
666 return value.BashArray(items)
667
668 elif op_name == 'keys': # "${!a[@]}"
669 keys = d.keys()
670 mylib.BigIntSort(keys)
671 items = [mops.ToStr(k) for k in keys]
672
673 # TODO: return SparseArray
674 return value.BashArray(items)
675
676 elif op_name == 'slice': # "${a[@]:0:5}"
677 start = rd.PosInt()
678 end = rd.PosInt()
679 rd.Done()
680
681 n = mops.BigTruncate(mops.Sub(end, start))
682 #log('start %d - end %d', start.i, end.i)
683
684 # Pre-allocate
685 items2 = [no_str] * n # type: List[str]
686
687 # Iterate from start to end. Note that this algorithm is
688 # theoretically slower than bash in the case where the array is
689 # sparse (in the part selected by the slice)
690 #
691 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
692 # elements, OSH will iterate through 999 integers and do 999 dict
693 # lookups, while bash will follow 3 pointers.
694 #
695 # However, in practice, I think iterating through integers is
696 # cheap.
697
698 j = 0
699 i = start
700 while mops.Greater(end, i): # i < end
701 s = d.get(i)
702 #log('s %s', s)
703 if s is not None:
704 items2[j] = s
705 j += 1
706
707 i = mops.Add(i, mops.ONE) # i += 1
708
709 # TODO: return SparseArray
710 return value.BashArray(items2)
711
712 elif op_name == 'append': # a+=(x y)
713 strs = rd.PosBashArray()
714
715 # TODO: We can maintain the max index in the value.SparseArray(),
716 # so that it's O(1) to append rather than O(n)
717 # - Update on 'set' is O(1)
718 # - Update on 'unset' is potentially O(n)
719
720 if 0:
721 max_index = mops.MINUS_ONE # Note: this works for empty arrays
722 for i1 in d:
723 if mops.Greater(i1, max_index): # i1 > max_index
724 max_index = i1
725 else:
726 max_index = sp.max_index
727
728 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
729 for s in strs:
730 d[i2] = s
731 i2 = mops.Add(i2, mops.ONE) # i2 += 1
732
733 # sp.max_index += len(strs)
734 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
735 return value.Int(mops.ZERO)
736
737 else:
738 print('Invalid SparseArray operation %r' % op_name)
739 return value.Int(mops.ZERO)