OILS / builtin / func_misc.py View on Github | oilshell.org

751 lines, 435 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str,
9 Dict_, Obj)
10
11from core import error
12from core import num
13from core import state
14from display import pp_value
15from display import ui
16from core import vm
17from data_lang import j8
18from frontend import match
19from frontend import typed_args
20from mycpp import mops
21from mycpp import mylib
22from mycpp.mylib import NewDict, iteritems, log, tagswitch
23from ysh import expr_eval
24from ysh import val_ops
25
26from typing import TYPE_CHECKING, Dict, List, Optional, cast
27if TYPE_CHECKING:
28 from osh import glob_
29 from osh import split
30
31_ = log
32
33
34class Object(vm._Callable):
35 """
36 Create an object. The order of params follows JavaScript's Object.create()
37
38 var obj = Object(prototype, props)
39 """
40
41 def __init__(self):
42 # type: () -> None
43 pass
44
45 def Call(self, rd):
46 # type: (typed_args.Reader) -> value_t
47
48 prototype = rd.PosValue()
49 props = rd.PosDict()
50 rd.Done()
51
52 chain = None # type: Optional[Obj]
53 UP_prototype = prototype
54 with tagswitch(prototype) as case:
55 if case(value_e.Null):
56 pass
57 elif case(value_e.Obj):
58 prototype = cast(Obj, UP_prototype)
59 chain = prototype
60 else:
61 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
62 rd.BlamePos())
63
64 # Opposite order
65 return Obj(props, chain)
66
67
68class Len(vm._Callable):
69
70 def __init__(self):
71 # type: () -> None
72 pass
73
74 def Call(self, rd):
75 # type: (typed_args.Reader) -> value_t
76
77 x = rd.PosValue()
78 rd.Done()
79
80 UP_x = x
81 with tagswitch(x) as case:
82 if case(value_e.List):
83 x = cast(value.List, UP_x)
84 return num.ToBig(len(x.items))
85
86 elif case(value_e.Dict):
87 x = cast(Dict_, UP_x)
88 return num.ToBig(len(x.d))
89
90 elif case(value_e.Str):
91 x = cast(value.Str, UP_x)
92 return num.ToBig(len(x.s))
93
94 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
95 rd.BlamePos())
96
97
98class Type(vm._Callable):
99
100 def __init__(self):
101 # type: () -> None
102 pass
103
104 def Call(self, rd):
105 # type: (typed_args.Reader) -> value_t
106
107 val = rd.PosValue()
108 rd.Done()
109
110 return value.Str(ui.ValType(val))
111
112
113class Join(vm._Callable):
114 """Both free function join() and List->join() method."""
115
116 def __init__(self):
117 # type: () -> None
118 pass
119
120 def Call(self, rd):
121 # type: (typed_args.Reader) -> value_t
122
123 li = rd.PosList()
124 delim = rd.OptionalStr(default_='')
125 rd.Done()
126
127 strs = [] # type: List[str]
128 for i, el in enumerate(li):
129 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
130
131 return value.Str(delim.join(strs))
132
133
134class Maybe(vm._Callable):
135
136 def __init__(self):
137 # type: () -> None
138 pass
139
140 def Call(self, rd):
141 # type: (typed_args.Reader) -> value_t
142
143 val = rd.PosValue()
144 rd.Done()
145
146 if val == value.Null:
147 return value.List([])
148
149 s = val_ops.ToStr(
150 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
151 rd.LeftParenToken())
152 if len(s):
153 return value.List([val]) # use val to avoid needlessly copy
154
155 return value.List([])
156
157
158class Bool(vm._Callable):
159
160 def __init__(self):
161 # type: () -> None
162 pass
163
164 def Call(self, rd):
165 # type: (typed_args.Reader) -> value_t
166
167 val = rd.PosValue()
168 rd.Done()
169
170 return value.Bool(val_ops.ToBool(val))
171
172
173class Int(vm._Callable):
174
175 def __init__(self):
176 # type: () -> None
177 pass
178
179 def Call(self, rd):
180 # type: (typed_args.Reader) -> value_t
181
182 val = rd.PosValue()
183 rd.Done()
184
185 UP_val = val
186 with tagswitch(val) as case:
187 if case(value_e.Int):
188 return val
189
190 elif case(value_e.Bool):
191 val = cast(value.Bool, UP_val)
192 return value.Int(mops.FromBool(val.b))
193
194 elif case(value_e.Float):
195 val = cast(value.Float, UP_val)
196 ok, big_int = mops.FromFloat(val.f)
197 if ok:
198 return value.Int(big_int)
199 else:
200 raise error.Expr(
201 "Can't convert float %s to Int" %
202 pp_value.FloatString(val.f), rd.BlamePos())
203
204 elif case(value_e.Str):
205 val = cast(value.Str, UP_val)
206 if not match.LooksLikeInteger(val.s):
207 raise error.Expr("Can't convert %s to Int" % val.s,
208 rd.BlamePos())
209
210 return value.Int(mops.FromStr(val.s))
211
212 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
213 rd.BlamePos())
214
215
216class Float(vm._Callable):
217
218 def __init__(self):
219 # type: () -> None
220 pass
221
222 def Call(self, rd):
223 # type: (typed_args.Reader) -> value_t
224
225 val = rd.PosValue()
226 rd.Done()
227
228 UP_val = val
229 with tagswitch(val) as case:
230 if case(value_e.Int):
231 val = cast(value.Int, UP_val)
232 return value.Float(mops.ToFloat(val.i))
233
234 elif case(value_e.Float):
235 return val
236
237 elif case(value_e.Str):
238 val = cast(value.Str, UP_val)
239 if not match.LooksLikeFloat(val.s):
240 raise error.Expr('Cannot convert %s to Float' % val.s,
241 rd.BlamePos())
242
243 return value.Float(float(val.s))
244
245 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
246 rd.BlamePos())
247
248
249class Str_(vm._Callable):
250
251 def __init__(self):
252 # type: () -> None
253 pass
254
255 def Call(self, rd):
256 # type: (typed_args.Reader) -> value_t
257
258 val = rd.PosValue()
259 rd.Done()
260
261 # TODO: Should we call Stringify here? That would handle Eggex.
262
263 UP_val = val
264 with tagswitch(val) as case:
265 if case(value_e.Int):
266 val = cast(value.Int, UP_val)
267 return value.Str(mops.ToStr(val.i))
268
269 elif case(value_e.Float):
270 val = cast(value.Float, UP_val)
271 return value.Str(str(val.f))
272
273 elif case(value_e.Str):
274 return val
275
276 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
277 rd.BlamePos())
278
279
280class List_(vm._Callable):
281
282 def __init__(self):
283 # type: () -> None
284 pass
285
286 def Call(self, rd):
287 # type: (typed_args.Reader) -> value_t
288
289 val = rd.PosValue()
290 rd.Done()
291
292 l = [] # type: List[value_t]
293 it = None # type: val_ops.Iterator
294 UP_val = val
295 with tagswitch(val) as case:
296 if case(value_e.List):
297 val = cast(value.List, UP_val)
298 it = val_ops.ListIterator(val)
299
300 elif case(value_e.Dict):
301 val = cast(Dict_, UP_val)
302 it = val_ops.DictIterator(val)
303
304 elif case(value_e.Range):
305 val = cast(value.Range, UP_val)
306 it = val_ops.RangeIterator(val)
307
308 else:
309 raise error.TypeErr(val,
310 'list() expected Dict, List, or Range',
311 rd.BlamePos())
312
313 assert it is not None
314 while True:
315 first = it.FirstValue()
316 if first is None:
317 break
318 l.append(first)
319 it.Next()
320
321 return value.List(l)
322
323
324class DictFunc(vm._Callable):
325
326 def __init__(self):
327 # type: () -> None
328 pass
329
330 def Call(self, rd):
331 # type: (typed_args.Reader) -> value_t
332
333 val = rd.PosValue()
334 rd.Done()
335
336 UP_val = val
337 with tagswitch(val) as case:
338 if case(value_e.Dict):
339 d = NewDict() # type: Dict[str, value_t]
340 val = cast(Dict_, UP_val)
341 for k, v in iteritems(val.d):
342 d[k] = v
343
344 return Dict_(d, None)
345
346 elif case(value_e.BashAssoc):
347 d = NewDict()
348 val = cast(value.BashAssoc, UP_val)
349 for k, s in iteritems(val.d):
350 d[k] = value.Str(s)
351
352 return Dict_(d, None)
353
354 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
355 rd.BlamePos())
356
357
358class Runes(vm._Callable):
359
360 def __init__(self):
361 # type: () -> None
362 pass
363
364 def Call(self, rd):
365 # type: (typed_args.Reader) -> value_t
366 return value.Null
367
368
369class EncodeRunes(vm._Callable):
370
371 def __init__(self):
372 # type: () -> None
373 pass
374
375 def Call(self, rd):
376 # type: (typed_args.Reader) -> value_t
377 return value.Null
378
379
380class Bytes(vm._Callable):
381
382 def __init__(self):
383 # type: () -> None
384 pass
385
386 def Call(self, rd):
387 # type: (typed_args.Reader) -> value_t
388 return value.Null
389
390
391class EncodeBytes(vm._Callable):
392
393 def __init__(self):
394 # type: () -> None
395 pass
396
397 def Call(self, rd):
398 # type: (typed_args.Reader) -> value_t
399 return value.Null
400
401
402class Split(vm._Callable):
403
404 def __init__(self, splitter):
405 # type: (split.SplitContext) -> None
406 vm._Callable.__init__(self)
407 self.splitter = splitter
408
409 def Call(self, rd):
410 # type: (typed_args.Reader) -> value_t
411 s = rd.PosStr()
412
413 ifs = rd.OptionalStr()
414
415 rd.Done()
416
417 l = [
418 value.Str(elem)
419 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
420 ] # type: List[value_t]
421 return value.List(l)
422
423
424class FloatsEqual(vm._Callable):
425
426 def __init__(self):
427 # type: () -> None
428 pass
429
430 def Call(self, rd):
431 # type: (typed_args.Reader) -> value_t
432 left = rd.PosFloat()
433 right = rd.PosFloat()
434 rd.Done()
435
436 return value.Bool(left == right)
437
438
439class Glob(vm._Callable):
440
441 def __init__(self, globber):
442 # type: (glob_.Globber) -> None
443 vm._Callable.__init__(self)
444 self.globber = globber
445
446 def Call(self, rd):
447 # type: (typed_args.Reader) -> value_t
448 s = rd.PosStr()
449 rd.Done()
450
451 out = [] # type: List[str]
452 self.globber._Glob(s, out)
453
454 l = [value.Str(elem) for elem in out] # type: List[value_t]
455 return value.List(l)
456
457
458class Shvar_get(vm._Callable):
459 """Look up with dynamic scope."""
460
461 def __init__(self, mem):
462 # type: (state.Mem) -> None
463 vm._Callable.__init__(self)
464 self.mem = mem
465
466 def Call(self, rd):
467 # type: (typed_args.Reader) -> value_t
468 name = rd.PosStr()
469 rd.Done()
470 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
471
472
473class GetVar(vm._Callable):
474 """Look up normal scoping rules."""
475
476 def __init__(self, mem):
477 # type: (state.Mem) -> None
478 vm._Callable.__init__(self)
479 self.mem = mem
480
481 def Call(self, rd):
482 # type: (typed_args.Reader) -> value_t
483 name = rd.PosStr()
484 rd.Done()
485 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
486
487
488class EvalExpr(vm._Callable):
489
490 def __init__(self, expr_ev):
491 # type: (expr_eval.ExprEvaluator) -> None
492 self.expr_ev = expr_ev
493
494 def Call(self, rd):
495 # type: (typed_args.Reader) -> value_t
496 lazy = rd.PosExpr()
497 rd.Done()
498
499 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
500
501 return result
502
503
504class ToJson8(vm._Callable):
505
506 def __init__(self, is_j8):
507 # type: (bool) -> None
508 self.is_j8 = is_j8
509
510 def Call(self, rd):
511 # type: (typed_args.Reader) -> value_t
512
513 val = rd.PosValue()
514 space = mops.BigTruncate(rd.NamedInt('space', 0))
515 rd.Done()
516
517 # Convert from external JS-like API to internal API.
518 if space <= 0:
519 indent = -1
520 else:
521 indent = space
522
523 buf = mylib.BufWriter()
524 try:
525 if self.is_j8:
526 j8.PrintMessage(val, buf, indent)
527 else:
528 j8.PrintJsonMessage(val, buf, indent)
529 except error.Encode as e:
530 # status code 4 is special, for encode/decode errors.
531 raise error.Structured(4, e.Message(), rd.LeftParenToken())
532
533 return value.Str(buf.getvalue())
534
535
536class FromJson8(vm._Callable):
537
538 def __init__(self, is_j8):
539 # type: (bool) -> None
540 self.is_j8 = is_j8
541
542 def Call(self, rd):
543 # type: (typed_args.Reader) -> value_t
544
545 s = rd.PosStr()
546 rd.Done()
547
548 p = j8.Parser(s, self.is_j8)
549 try:
550 val = p.ParseValue()
551 except error.Decode as e:
552 # Right now I'm not exposing the original string, because that
553 # could lead to a memory leak in the _error Dict.
554 # The message quotes part of the string, and we could improve
555 # that. We could have a substring with context.
556 props = {
557 'start_pos': num.ToBig(e.start_pos),
558 'end_pos': num.ToBig(e.end_pos),
559 } # type: Dict[str, value_t]
560 # status code 4 is special, for encode/decode errors.
561 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
562
563 return val
564
565
566class BashArrayToSparse(vm._Callable):
567 """
568 value.BashArray -> value.SparseArray, for testing
569 """
570
571 def __init__(self):
572 # type: () -> None
573 pass
574
575 def Call(self, rd):
576 # type: (typed_args.Reader) -> value_t
577
578 strs = rd.PosBashArray()
579 rd.Done()
580
581 d = {} # type: Dict[mops.BigInt, str]
582 max_index = mops.MINUS_ONE # max index for empty array
583 for i, s in enumerate(strs):
584 if s is not None:
585 big_i = mops.IntWiden(i)
586 d[big_i] = s
587 if mops.Greater(big_i, max_index):
588 max_index = big_i
589
590 return value.SparseArray(d, max_index)
591
592
593class SparseOp(vm._Callable):
594 """
595 All ops on value.SparseArray, for testing performance
596 """
597
598 def __init__(self):
599 # type: () -> None
600 pass
601
602 def Call(self, rd):
603 # type: (typed_args.Reader) -> value_t
604
605 sp = rd.PosSparseArray()
606 d = sp.d
607 #i = mops.BigTruncate(rd.PosInt())
608 op_name = rd.PosStr()
609
610 no_str = None # type: str
611
612 if op_name == 'len': # ${#a[@]}
613 rd.Done()
614 return num.ToBig(len(d))
615
616 elif op_name == 'get': # ${a[42]}
617 index = rd.PosInt()
618 rd.Done()
619
620 s = d.get(index)
621 if s is None:
622 return value.Null
623 else:
624 return value.Str(s)
625
626 elif op_name == 'set': # a[42]=foo
627 index = rd.PosInt()
628 s = rd.PosStr()
629 rd.Done()
630
631 d[index] = s
632
633 if mops.Greater(index, sp.max_index):
634 sp.max_index = index
635
636 return value.Int(mops.ZERO)
637
638 elif op_name == 'unset': # unset 'a[1]'
639 index = rd.PosInt()
640 rd.Done()
641
642 mylib.dict_erase(d, index)
643
644 max_index = mops.MINUS_ONE # Note: this works if d is not empty
645 for i1 in d:
646 if mops.Greater(i1, max_index): # i1 > max_index
647 max_index = i1
648 sp.max_index = max_index
649
650 return value.Int(mops.ZERO)
651
652 elif op_name == 'subst': # "${a[@]}"
653 # Algorithm to expand a Dict[BigInt, Str]
654 #
655 # 1. Copy the integer keys into a new List
656 # 2. Sort them in numeric order
657 # 3. Create a List[str] that's the same size as the keys
658 # 4. Loop through sorted keys, look up value, and populate list
659 #
660 # There is another possible algorithm:
661 #
662 # 1. Copy the VALUES into a new list
663 # 2. Somehow sort them by the CORRESPONDING key, which depends on
664 # Slab<> POSITION. I think this does not fit within the
665 # std::sort() model. I think we would have to write a little custom
666 # sort algorithm.
667
668 keys = d.keys()
669 mylib.BigIntSort(keys)
670 # Pre-allocate
671 items = [no_str] * len(d) # type: List[str]
672 j = 0
673 for i in keys:
674 s = d.get(i)
675 assert s is not None
676 items[j] = s
677 j += 1
678 return value.BashArray(items)
679
680 elif op_name == 'keys': # "${!a[@]}"
681 keys = d.keys()
682 mylib.BigIntSort(keys)
683 items = [mops.ToStr(k) for k in keys]
684
685 # TODO: return SparseArray
686 return value.BashArray(items)
687
688 elif op_name == 'slice': # "${a[@]:0:5}"
689 start = rd.PosInt()
690 end = rd.PosInt()
691 rd.Done()
692
693 n = mops.BigTruncate(mops.Sub(end, start))
694 #log('start %d - end %d', start.i, end.i)
695
696 # Pre-allocate
697 items2 = [no_str] * n # type: List[str]
698
699 # Iterate from start to end. Note that this algorithm is
700 # theoretically slower than bash in the case where the array is
701 # sparse (in the part selected by the slice)
702 #
703 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
704 # elements, OSH will iterate through 999 integers and do 999 dict
705 # lookups, while bash will follow 3 pointers.
706 #
707 # However, in practice, I think iterating through integers is
708 # cheap.
709
710 j = 0
711 i = start
712 while mops.Greater(end, i): # i < end
713 s = d.get(i)
714 #log('s %s', s)
715 if s is not None:
716 items2[j] = s
717 j += 1
718
719 i = mops.Add(i, mops.ONE) # i += 1
720
721 # TODO: return SparseArray
722 return value.BashArray(items2)
723
724 elif op_name == 'append': # a+=(x y)
725 strs = rd.PosBashArray()
726
727 # TODO: We can maintain the max index in the value.SparseArray(),
728 # so that it's O(1) to append rather than O(n)
729 # - Update on 'set' is O(1)
730 # - Update on 'unset' is potentially O(n)
731
732 if 0:
733 max_index = mops.MINUS_ONE # Note: this works for empty arrays
734 for i1 in d:
735 if mops.Greater(i1, max_index): # i1 > max_index
736 max_index = i1
737 else:
738 max_index = sp.max_index
739
740 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
741 for s in strs:
742 d[i2] = s
743 i2 = mops.Add(i2, mops.ONE) # i2 += 1
744
745 # sp.max_index += len(strs)
746 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
747 return value.Int(mops.ZERO)
748
749 else:
750 print('Invalid SparseArray operation %r' % op_name)
751 return value.Int(mops.ZERO)