OILS / builtin / func_misc.py View on Github | oilshell.org

717 lines, 416 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.runtime_asdl import (scope_e)
8from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str,
9 Dict_)
10
11from core import error
12from core import num
13from core import state
14from display import pp_value
15from display import ui
16from core import vm
17from data_lang import j8
18from frontend import match
19from frontend import typed_args
20from mycpp import mops
21from mycpp import mylib
22from mycpp.mylib import NewDict, iteritems, log, tagswitch
23from ysh import expr_eval
24from ysh import val_ops
25
26from typing import TYPE_CHECKING, Dict, List, cast
27if TYPE_CHECKING:
28 from osh import glob_
29 from osh import split
30
31_ = log
32
33
34class Len(vm._Callable):
35
36 def __init__(self):
37 # type: () -> None
38 pass
39
40 def Call(self, rd):
41 # type: (typed_args.Reader) -> value_t
42
43 x = rd.PosValue()
44 rd.Done()
45
46 UP_x = x
47 with tagswitch(x) as case:
48 if case(value_e.List):
49 x = cast(value.List, UP_x)
50 return num.ToBig(len(x.items))
51
52 elif case(value_e.Dict):
53 x = cast(Dict_, UP_x)
54 return num.ToBig(len(x.d))
55
56 elif case(value_e.Str):
57 x = cast(value.Str, UP_x)
58 return num.ToBig(len(x.s))
59
60 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
61 rd.BlamePos())
62
63
64class Type(vm._Callable):
65
66 def __init__(self):
67 # type: () -> None
68 pass
69
70 def Call(self, rd):
71 # type: (typed_args.Reader) -> value_t
72
73 val = rd.PosValue()
74 rd.Done()
75
76 return value.Str(ui.ValType(val))
77
78
79class Join(vm._Callable):
80 """Both free function join() and List->join() method."""
81
82 def __init__(self):
83 # type: () -> None
84 pass
85
86 def Call(self, rd):
87 # type: (typed_args.Reader) -> value_t
88
89 li = rd.PosList()
90 delim = rd.OptionalStr(default_='')
91 rd.Done()
92
93 strs = [] # type: List[str]
94 for i, el in enumerate(li):
95 strs.append(val_ops.Stringify(el, rd.LeftParenToken()))
96
97 return value.Str(delim.join(strs))
98
99
100class Maybe(vm._Callable):
101
102 def __init__(self):
103 # type: () -> None
104 pass
105
106 def Call(self, rd):
107 # type: (typed_args.Reader) -> value_t
108
109 val = rd.PosValue()
110 rd.Done()
111
112 if val == value.Null:
113 return value.List([])
114
115 s = val_ops.ToStr(
116 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
117 rd.LeftParenToken())
118 if len(s):
119 return value.List([val]) # use val to avoid needlessly copy
120
121 return value.List([])
122
123
124class Bool(vm._Callable):
125
126 def __init__(self):
127 # type: () -> None
128 pass
129
130 def Call(self, rd):
131 # type: (typed_args.Reader) -> value_t
132
133 val = rd.PosValue()
134 rd.Done()
135
136 return value.Bool(val_ops.ToBool(val))
137
138
139class Int(vm._Callable):
140
141 def __init__(self):
142 # type: () -> None
143 pass
144
145 def Call(self, rd):
146 # type: (typed_args.Reader) -> value_t
147
148 val = rd.PosValue()
149 rd.Done()
150
151 UP_val = val
152 with tagswitch(val) as case:
153 if case(value_e.Int):
154 return val
155
156 elif case(value_e.Bool):
157 val = cast(value.Bool, UP_val)
158 return value.Int(mops.FromBool(val.b))
159
160 elif case(value_e.Float):
161 val = cast(value.Float, UP_val)
162 ok, big_int = mops.FromFloat(val.f)
163 if ok:
164 return value.Int(big_int)
165 else:
166 raise error.Expr(
167 "Can't convert float %s to Int" %
168 pp_value.FloatString(val.f), rd.BlamePos())
169
170 elif case(value_e.Str):
171 val = cast(value.Str, UP_val)
172 if not match.LooksLikeInteger(val.s):
173 raise error.Expr("Can't convert %s to Int" % val.s,
174 rd.BlamePos())
175
176 return value.Int(mops.FromStr(val.s))
177
178 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
179 rd.BlamePos())
180
181
182class Float(vm._Callable):
183
184 def __init__(self):
185 # type: () -> None
186 pass
187
188 def Call(self, rd):
189 # type: (typed_args.Reader) -> value_t
190
191 val = rd.PosValue()
192 rd.Done()
193
194 UP_val = val
195 with tagswitch(val) as case:
196 if case(value_e.Int):
197 val = cast(value.Int, UP_val)
198 return value.Float(mops.ToFloat(val.i))
199
200 elif case(value_e.Float):
201 return val
202
203 elif case(value_e.Str):
204 val = cast(value.Str, UP_val)
205 if not match.LooksLikeFloat(val.s):
206 raise error.Expr('Cannot convert %s to Float' % val.s,
207 rd.BlamePos())
208
209 return value.Float(float(val.s))
210
211 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
212 rd.BlamePos())
213
214
215class Str_(vm._Callable):
216
217 def __init__(self):
218 # type: () -> None
219 pass
220
221 def Call(self, rd):
222 # type: (typed_args.Reader) -> value_t
223
224 val = rd.PosValue()
225 rd.Done()
226
227 # TODO: Should we call Stringify here? That would handle Eggex.
228
229 UP_val = val
230 with tagswitch(val) as case:
231 if case(value_e.Int):
232 val = cast(value.Int, UP_val)
233 return value.Str(mops.ToStr(val.i))
234
235 elif case(value_e.Float):
236 val = cast(value.Float, UP_val)
237 return value.Str(str(val.f))
238
239 elif case(value_e.Str):
240 return val
241
242 raise error.TypeErr(val, 'str() expected Str, Int, or Float',
243 rd.BlamePos())
244
245
246class List_(vm._Callable):
247
248 def __init__(self):
249 # type: () -> None
250 pass
251
252 def Call(self, rd):
253 # type: (typed_args.Reader) -> value_t
254
255 val = rd.PosValue()
256 rd.Done()
257
258 l = [] # type: List[value_t]
259 it = None # type: val_ops.Iterator
260 UP_val = val
261 with tagswitch(val) as case:
262 if case(value_e.List):
263 val = cast(value.List, UP_val)
264 it = val_ops.ListIterator(val)
265
266 elif case(value_e.Dict):
267 val = cast(Dict_, UP_val)
268 it = val_ops.DictIterator(val)
269
270 elif case(value_e.Range):
271 val = cast(value.Range, UP_val)
272 it = val_ops.RangeIterator(val)
273
274 else:
275 raise error.TypeErr(val,
276 'list() expected Dict, List, or Range',
277 rd.BlamePos())
278
279 assert it is not None
280 while True:
281 first = it.FirstValue()
282 if first is None:
283 break
284 l.append(first)
285 it.Next()
286
287 return value.List(l)
288
289
290class DictFunc(vm._Callable):
291
292 def __init__(self):
293 # type: () -> None
294 pass
295
296 def Call(self, rd):
297 # type: (typed_args.Reader) -> value_t
298
299 val = rd.PosValue()
300 rd.Done()
301
302 UP_val = val
303 with tagswitch(val) as case:
304 if case(value_e.Dict):
305 d = NewDict() # type: Dict[str, value_t]
306 val = cast(Dict_, UP_val)
307 for k, v in iteritems(val.d):
308 d[k] = v
309
310 return Dict_(d, None)
311
312 elif case(value_e.BashAssoc):
313 d = NewDict()
314 val = cast(value.BashAssoc, UP_val)
315 for k, s in iteritems(val.d):
316 d[k] = value.Str(s)
317
318 return Dict_(d, None)
319
320 raise error.TypeErr(val, 'dict() expected Dict or BashAssoc',
321 rd.BlamePos())
322
323
324class Runes(vm._Callable):
325
326 def __init__(self):
327 # type: () -> None
328 pass
329
330 def Call(self, rd):
331 # type: (typed_args.Reader) -> value_t
332 return value.Null
333
334
335class EncodeRunes(vm._Callable):
336
337 def __init__(self):
338 # type: () -> None
339 pass
340
341 def Call(self, rd):
342 # type: (typed_args.Reader) -> value_t
343 return value.Null
344
345
346class Bytes(vm._Callable):
347
348 def __init__(self):
349 # type: () -> None
350 pass
351
352 def Call(self, rd):
353 # type: (typed_args.Reader) -> value_t
354 return value.Null
355
356
357class EncodeBytes(vm._Callable):
358
359 def __init__(self):
360 # type: () -> None
361 pass
362
363 def Call(self, rd):
364 # type: (typed_args.Reader) -> value_t
365 return value.Null
366
367
368class Split(vm._Callable):
369
370 def __init__(self, splitter):
371 # type: (split.SplitContext) -> None
372 vm._Callable.__init__(self)
373 self.splitter = splitter
374
375 def Call(self, rd):
376 # type: (typed_args.Reader) -> value_t
377 s = rd.PosStr()
378
379 ifs = rd.OptionalStr()
380
381 rd.Done()
382
383 l = [
384 value.Str(elem)
385 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
386 ] # type: List[value_t]
387 return value.List(l)
388
389
390class FloatsEqual(vm._Callable):
391
392 def __init__(self):
393 # type: () -> None
394 pass
395
396 def Call(self, rd):
397 # type: (typed_args.Reader) -> value_t
398 left = rd.PosFloat()
399 right = rd.PosFloat()
400 rd.Done()
401
402 return value.Bool(left == right)
403
404
405class Glob(vm._Callable):
406
407 def __init__(self, globber):
408 # type: (glob_.Globber) -> None
409 vm._Callable.__init__(self)
410 self.globber = globber
411
412 def Call(self, rd):
413 # type: (typed_args.Reader) -> value_t
414 s = rd.PosStr()
415 rd.Done()
416
417 out = [] # type: List[str]
418 self.globber._Glob(s, out)
419
420 l = [value.Str(elem) for elem in out] # type: List[value_t]
421 return value.List(l)
422
423
424class Shvar_get(vm._Callable):
425 """Look up with dynamic scope."""
426
427 def __init__(self, mem):
428 # type: (state.Mem) -> None
429 vm._Callable.__init__(self)
430 self.mem = mem
431
432 def Call(self, rd):
433 # type: (typed_args.Reader) -> value_t
434 name = rd.PosStr()
435 rd.Done()
436 return state.DynamicGetVar(self.mem, name, scope_e.Dynamic)
437
438
439class GetVar(vm._Callable):
440 """Look up normal scoping rules."""
441
442 def __init__(self, mem):
443 # type: (state.Mem) -> None
444 vm._Callable.__init__(self)
445 self.mem = mem
446
447 def Call(self, rd):
448 # type: (typed_args.Reader) -> value_t
449 name = rd.PosStr()
450 rd.Done()
451 return state.DynamicGetVar(self.mem, name, scope_e.LocalOrGlobal)
452
453
454class EvalExpr(vm._Callable):
455
456 def __init__(self, expr_ev):
457 # type: (expr_eval.ExprEvaluator) -> None
458 self.expr_ev = expr_ev
459
460 def Call(self, rd):
461 # type: (typed_args.Reader) -> value_t
462 lazy = rd.PosExpr()
463 rd.Done()
464
465 result = self.expr_ev.EvalExpr(lazy, rd.LeftParenToken())
466
467 return result
468
469
470class ToJson8(vm._Callable):
471
472 def __init__(self, is_j8):
473 # type: (bool) -> None
474 self.is_j8 = is_j8
475
476 def Call(self, rd):
477 # type: (typed_args.Reader) -> value_t
478
479 val = rd.PosValue()
480 space = mops.BigTruncate(rd.NamedInt('space', 0))
481 rd.Done()
482
483 # Convert from external JS-like API to internal API.
484 if space <= 0:
485 indent = -1
486 else:
487 indent = space
488
489 buf = mylib.BufWriter()
490 try:
491 if self.is_j8:
492 j8.PrintMessage(val, buf, indent)
493 else:
494 j8.PrintJsonMessage(val, buf, indent)
495 except error.Encode as e:
496 # status code 4 is special, for encode/decode errors.
497 raise error.Structured(4, e.Message(), rd.LeftParenToken())
498
499 return value.Str(buf.getvalue())
500
501
502class FromJson8(vm._Callable):
503
504 def __init__(self, is_j8):
505 # type: (bool) -> None
506 self.is_j8 = is_j8
507
508 def Call(self, rd):
509 # type: (typed_args.Reader) -> value_t
510
511 s = rd.PosStr()
512 rd.Done()
513
514 p = j8.Parser(s, self.is_j8)
515 try:
516 val = p.ParseValue()
517 except error.Decode as e:
518 # Right now I'm not exposing the original string, because that
519 # could lead to a memory leak in the _error Dict.
520 # The message quotes part of the string, and we could improve
521 # that. We could have a substring with context.
522 props = {
523 'start_pos': num.ToBig(e.start_pos),
524 'end_pos': num.ToBig(e.end_pos),
525 } # type: Dict[str, value_t]
526 # status code 4 is special, for encode/decode errors.
527 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
528
529 return val
530
531
532class BashArrayToSparse(vm._Callable):
533 """
534 value.BashArray -> value.SparseArray, for testing
535 """
536
537 def __init__(self):
538 # type: () -> None
539 pass
540
541 def Call(self, rd):
542 # type: (typed_args.Reader) -> value_t
543
544 strs = rd.PosBashArray()
545 rd.Done()
546
547 d = {} # type: Dict[mops.BigInt, str]
548 max_index = mops.MINUS_ONE # max index for empty array
549 for i, s in enumerate(strs):
550 if s is not None:
551 big_i = mops.IntWiden(i)
552 d[big_i] = s
553 if mops.Greater(big_i, max_index):
554 max_index = big_i
555
556 return value.SparseArray(d, max_index)
557
558
559class SparseOp(vm._Callable):
560 """
561 All ops on value.SparseArray, for testing performance
562 """
563
564 def __init__(self):
565 # type: () -> None
566 pass
567
568 def Call(self, rd):
569 # type: (typed_args.Reader) -> value_t
570
571 sp = rd.PosSparseArray()
572 d = sp.d
573 #i = mops.BigTruncate(rd.PosInt())
574 op_name = rd.PosStr()
575
576 no_str = None # type: str
577
578 if op_name == 'len': # ${#a[@]}
579 rd.Done()
580 return num.ToBig(len(d))
581
582 elif op_name == 'get': # ${a[42]}
583 index = rd.PosInt()
584 rd.Done()
585
586 s = d.get(index)
587 if s is None:
588 return value.Null
589 else:
590 return value.Str(s)
591
592 elif op_name == 'set': # a[42]=foo
593 index = rd.PosInt()
594 s = rd.PosStr()
595 rd.Done()
596
597 d[index] = s
598
599 if mops.Greater(index, sp.max_index):
600 sp.max_index = index
601
602 return value.Int(mops.ZERO)
603
604 elif op_name == 'unset': # unset 'a[1]'
605 index = rd.PosInt()
606 rd.Done()
607
608 mylib.dict_erase(d, index)
609
610 max_index = mops.MINUS_ONE # Note: this works if d is not empty
611 for i1 in d:
612 if mops.Greater(i1, max_index): # i1 > max_index
613 max_index = i1
614 sp.max_index = max_index
615
616 return value.Int(mops.ZERO)
617
618 elif op_name == 'subst': # "${a[@]}"
619 # Algorithm to expand a Dict[BigInt, Str]
620 #
621 # 1. Copy the integer keys into a new List
622 # 2. Sort them in numeric order
623 # 3. Create a List[str] that's the same size as the keys
624 # 4. Loop through sorted keys, look up value, and populate list
625 #
626 # There is another possible algorithm:
627 #
628 # 1. Copy the VALUES into a new list
629 # 2. Somehow sort them by the CORRESPONDING key, which depends on
630 # Slab<> POSITION. I think this does not fit within the
631 # std::sort() model. I think we would have to write a little custom
632 # sort algorithm.
633
634 keys = d.keys()
635 mylib.BigIntSort(keys)
636 # Pre-allocate
637 items = [no_str] * len(d) # type: List[str]
638 j = 0
639 for i in keys:
640 s = d.get(i)
641 assert s is not None
642 items[j] = s
643 j += 1
644 return value.BashArray(items)
645
646 elif op_name == 'keys': # "${!a[@]}"
647 keys = d.keys()
648 mylib.BigIntSort(keys)
649 items = [mops.ToStr(k) for k in keys]
650
651 # TODO: return SparseArray
652 return value.BashArray(items)
653
654 elif op_name == 'slice': # "${a[@]:0:5}"
655 start = rd.PosInt()
656 end = rd.PosInt()
657 rd.Done()
658
659 n = mops.BigTruncate(mops.Sub(end, start))
660 #log('start %d - end %d', start.i, end.i)
661
662 # Pre-allocate
663 items2 = [no_str] * n # type: List[str]
664
665 # Iterate from start to end. Note that this algorithm is
666 # theoretically slower than bash in the case where the array is
667 # sparse (in the part selected by the slice)
668 #
669 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
670 # elements, OSH will iterate through 999 integers and do 999 dict
671 # lookups, while bash will follow 3 pointers.
672 #
673 # However, in practice, I think iterating through integers is
674 # cheap.
675
676 j = 0
677 i = start
678 while mops.Greater(end, i): # i < end
679 s = d.get(i)
680 #log('s %s', s)
681 if s is not None:
682 items2[j] = s
683 j += 1
684
685 i = mops.Add(i, mops.ONE) # i += 1
686
687 # TODO: return SparseArray
688 return value.BashArray(items2)
689
690 elif op_name == 'append': # a+=(x y)
691 strs = rd.PosBashArray()
692
693 # TODO: We can maintain the max index in the value.SparseArray(),
694 # so that it's O(1) to append rather than O(n)
695 # - Update on 'set' is O(1)
696 # - Update on 'unset' is potentially O(n)
697
698 if 0:
699 max_index = mops.MINUS_ONE # Note: this works for empty arrays
700 for i1 in d:
701 if mops.Greater(i1, max_index): # i1 > max_index
702 max_index = i1
703 else:
704 max_index = sp.max_index
705
706 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
707 for s in strs:
708 d[i2] = s
709 i2 = mops.Add(i2, mops.ONE) # i2 += 1
710
711 # sp.max_index += len(strs)
712 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
713 return value.Int(mops.ZERO)
714
715 else:
716 print('Invalid SparseArray operation %r' % op_name)
717 return value.Int(mops.ZERO)