OILS / spec / ysh-json.test.sh View on Github | oilshell.org

1036 lines, 270 significant
1## oils_failures_allowed: 1
2## tags: dev-minimal
3
4#### usage errors
5
6json read zz
7echo status=$?
8
9json write
10
11## status: 3
12## STDOUT:
13status=2
14## END
15
16#### json write STRING
17shopt --set parse_proc
18
19json write ('foo')
20var s = 'foo'
21json write (s)
22## STDOUT:
23"foo"
24"foo"
25## END
26
27#### json write ARRAY
28json write (:|foo.cc foo.h|)
29json write (['foo.cc', 'foo.h'], space=0)
30## STDOUT:
31[
32 "foo.cc",
33 "foo.h"
34]
35["foo.cc","foo.h"]
36## END
37
38#### json write Dict
39json write ({k: 'v', k2: [4, 5]})
40
41json write ([{k: 'v', k2: 'v2'}, {}])
42
43## STDOUT:
44{
45 "k": "v",
46 "k2": [
47 4,
48 5
49 ]
50}
51[
52 {
53 "k": "v",
54 "k2": "v2"
55 },
56 {}
57]
58## END
59
60#### json write space=0, space=4
61shopt --set parse_proc
62
63var mydict = {name: "bob", age: 30}
64
65json write (mydict, space=0)
66json write (mydict, space=4)
67## STDOUT:
68{"name":"bob","age":30}
69{
70 "name": "bob",
71 "age": 30
72}
73## END
74
75#### json write in command sub
76shopt -s oil:all # for echo
77var mydict = {name: "bob", age: 30}
78json write (mydict)
79var x = $(json write (mydict))
80echo $x
81## STDOUT:
82{
83 "name": "bob",
84 "age": 30
85}
86{
87 "name": "bob",
88 "age": 30
89}
90## END
91
92#### json read passed invalid args
93
94# EOF
95json read
96echo status=$?
97
98json read 'z z'
99echo status=$?
100
101json read a b c
102echo status=$?
103
104## STDOUT:
105status=1
106status=2
107status=2
108## END
109
110#### json read uses $_reply var
111
112# space before true
113echo ' true' | json read
114json write (_reply)
115
116## STDOUT:
117true
118## END
119
120#### json read then json write
121
122# BUG with space before true
123echo '{"name": "bob", "age": 42, "ok": true}' | json read
124json write (_reply)
125
126echo '{"name": "bob", "age": 42, "ok":true}' | json read
127json write (_reply)
128
129echo '{"name": {}, "age": {}, "x":-1, "y": -0}' | json read
130json write (_reply)
131
132## STDOUT:
133{
134 "name": "bob",
135 "age": 42,
136 "ok": true
137}
138{
139 "name": "bob",
140 "age": 42,
141 "ok": true
142}
143{
144 "name": {},
145 "age": {},
146 "x": -1,
147 "y": 0
148}
149## END
150
151#### json read with redirect
152echo '{"age": 42}' > $TMP/foo.txt
153json read (&x) < $TMP/foo.txt
154pp cell :x
155## STDOUT:
156x = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:42)]))
157## END
158
159#### json read at end of pipeline (relies on lastpipe)
160echo '{"age": 43}' | json read (&y)
161pp cell y
162## STDOUT:
163y = (Cell exported:F readonly:F nameref:F val:(value.Dict d:[Dict age (value.Int i:43)]))
164## END
165
166#### invalid JSON
167echo '{' | json read (&y)
168echo pipeline status = $?
169pp line (y)
170## status: 1
171## STDOUT:
172pipeline status = 1
173## END
174
175#### Extra data after valid JSON
176
177# Trailing space is OK
178echo '42 ' | json read
179echo num space $?
180
181echo '{} ' | json read
182echo obj space $?
183
184echo '42 # comment' | json8 read
185echo num comment $?
186
187echo '{} # comment ' | json8 read
188echo obj comment $?
189
190echo '42]' | json read
191echo num bracket $?
192
193echo '{}]' | json read
194echo obj bracket $?
195
196## STDOUT:
197num space 0
198obj space 0
199num comment 0
200obj comment 0
201num bracket 1
202obj bracket 1
203## END
204
205#### json write expression
206json write ([1,2,3], space=0)
207echo status=$?
208
209json write (5, 6) # to many args
210echo status=$?
211
212## status: 3
213## STDOUT:
214[1,2,3]
215status=0
216## END
217
218#### json write evaluation error
219
220#var block = ^(echo hi)
221#json write (block)
222#echo status=$?
223
224# undefined var
225json write (a)
226echo 'should have failed'
227
228## status: 1
229## STDOUT:
230## END
231
232#### json write of List in cycle
233
234var L = [1, 2, 3]
235setvar L[0] = L
236
237shopt -s ysh:upgrade
238fopen >tmp.txt {
239 pp line (L)
240}
241fgrep -n -o '[ -->' tmp.txt
242
243json write (L)
244echo 'should have failed'
245
246## status: 1
247## STDOUT:
2481:[ -->
249## END
250
251#### json write of Dict in cycle
252
253var d = {}
254setvar d.k = d
255
256shopt -s ysh:upgrade
257fopen >tmp.txt {
258 pp line (d)
259}
260fgrep -n -o '{ -->' tmp.txt
261
262json write (d)
263echo 'should have failed'
264
265## status: 1
266## STDOUT:
2671:{ -->
268## END
269
270#### json write of List/Dict referenced twice (bug fix)
271
272var mylist = [1,2,3]
273var mydict = {foo: "bar"}
274
275var top = {k: mylist, k2: mylist, k3: mydict, k4: mydict}
276
277# BUG!
278json write (top, space=0)
279
280## STDOUT:
281{"k":[1,2,3],"k2":[1,2,3],"k3":{"foo":"bar"},"k4":{"foo":"bar"}}
282## END
283
284#### json read doesn't accept u'' or b'' strings
285
286json read <<EOF
287{"key": u'val'}
288EOF
289echo status=$?
290
291#pp line (_reply)
292
293json read <<EOF
294{"key": b'val'}
295EOF
296echo status=$?
297
298## STDOUT:
299status=1
300status=1
301## END
302
303#### json read doesn't accept comments, but json8 does
304
305json8 read <<EOF
306{ # comment
307 "key": # zz
308 b'val', # yy
309 "k2": "v2" #
310}
311EOF
312echo status=$?
313
314json8 write (_reply)
315
316json read <<EOF
317{"key": "val"} # comment
318EOF
319echo status=$?
320## STDOUT:
321status=0
322{
323 "key": "val",
324 "k2": "v2"
325}
326status=1
327## END
328
329
330#### json write emits Unicode replacement char for binary data \yff
331
332json write ([3, "foo", $'-\xff\xfe---\xfd=']) > tmp.txt
333
334# Round trip it for good measure
335json read < tmp.txt
336
337json write (_reply)
338
339## STDOUT:
340[
341 3,
342 "foo",
343 "-��---�="
344]
345## END
346
347#### json8 write emits b'' strings for binary data \yff
348
349json8 write ([3, "foo", $'-\xff\xfe-\xfd='])
350
351## STDOUT:
352[
353 3,
354 "foo",
355 b'-\yff\yfe-\yfd='
356]
357## END
358
359
360#### json8 write bytes vs unicode string
361
362u=$'mu \u03bc \x01 \" \\ \b\f\n\r\t'
363u2=$'\x01\x1f' # this is a valid unicode string
364
365b=$'\xff' # this isn't valid unicode
366
367json8 write (u)
368json8 write (u2)
369
370json8 write (b)
371
372## STDOUT:
373"mu μ \u0001 \" \\ \b\f\n\r\t"
374"\u0001\u001f"
375b'\yff'
376## END
377
378#### JSON \/ escapes supported
379
380msg='"\/"'
381
382echo "$msg" | python3 -c 'import json, sys; print(json.load(sys.stdin))'
383
384echo "$msg" | json read
385echo reply=$_reply
386
387j8="b'\\/'"
388echo "$msg" | json read
389echo reply=$_reply
390
391
392## STDOUT:
393/
394reply=/
395reply=/
396## END
397
398#### JSON string can have unescaped ' and J8 string can have unescaped "
399
400json read <<EOF
401"'"
402EOF
403
404pp line (_reply)
405
406json8 read <<EOF
407u'"'
408EOF
409
410pp line (_reply)
411
412## STDOUT:
413(Str) "'"
414(Str) "\""
415## END
416
417
418#### J8 supports superfluous \" escapes, but JSON doesn't support \' escapes
419
420json8 read <<'EOF'
421b'\"'
422EOF
423echo reply=$_reply
424
425json8 read <<'EOF'
426b'\'\'\b\f\n\r\t\"\\'
427EOF
428pp line (_reply)
429
430# Suppress traceback
431python3 -c 'import json, sys; print(json.load(sys.stdin))' 2>/dev/null <<'EOF'
432"\'"
433EOF
434echo python3=$?
435
436json read <<'EOF'
437"\'"
438EOF
439echo json=$?
440
441## STDOUT:
442reply="
443(Str) "''\b\f\n\r\t\"\\"
444python3=1
445json=1
446## END
447
448#### Escaping uses \u0001 in "", but \u{1} in b''
449
450s1=$'\x01'
451s2=$'\x01\xff\x1f' # byte string
452
453json8 write (s1)
454json8 write (s2)
455
456## STDOUT:
457"\u0001"
458b'\u{1}\yff\u{1f}'
459## END
460
461
462#### json8 read
463
464echo '{ }' | json8 read
465pp line (_reply)
466
467echo '[ ]' | json8 read
468pp line (_reply)
469
470echo '[42]' | json8 read
471pp line (_reply)
472
473echo '[true, false]' | json8 read
474pp line (_reply)
475
476echo '{"k": "v"}' | json8 read
477pp line (_reply)
478
479echo '{"k": null}' | json8 read
480pp line (_reply)
481
482echo '{"k": 1, "k2": 2}' | json8 read
483pp line (_reply)
484
485echo "{u'k': {b'k2': null}}" | json8 read
486pp line (_reply)
487
488echo '{"k": {"k2": "v2"}, "k3": "backslash \\ \" \n line 2 \u03bc "}' | json8 read
489pp line (_reply)
490
491json8 read (&x) <<'EOF'
492{u'k': {u'k2': u'v2'}, u'k3': u'backslash \\ \" \n line 2 \u{3bc} '}
493EOF
494pp line (x)
495
496## STDOUT:
497(Dict) {}
498(List) []
499(List) [42]
500(List) [true,false]
501(Dict) {"k":"v"}
502(Dict) {"k":null}
503(Dict) {"k":1,"k2":2}
504(Dict) {"k":{"k2":null}}
505(Dict) {"k":{"k2":"v2"},"k3":"backslash \\ \" \n line 2 μ "}
506(Dict) {"k":{"k2":"v2"},"k3":"backslash \\ \" \n line 2 μ "}
507## END
508
509#### json8 round trip
510
511var obj = [42, 1.5, null, true, "hi", b'\yff\yfe\b\n""']
512
513json8 write (obj, space=0) > j
514
515cat j
516
517json8 read < j
518
519json8 write (_reply)
520
521## STDOUT:
522[42,1.5,null,true,"hi",b'\yff\yfe\b\n""']
523[
524 42,
525 1.5,
526 null,
527 true,
528 "hi",
529 b'\yff\yfe\b\n""'
530]
531## END
532
533#### json round trip (regression)
534
535var d = {
536 short: '-v', long: '--verbose', type: null, default: '', help: 'Enable verbose logging'
537}
538
539json write (d) | json read
540
541pp line (_reply)
542
543## STDOUT:
544(Dict) {"short":"-v","long":"--verbose","type":null,"default":"","help":"Enable verbose logging"}
545## END
546
547#### round trip: decode surrogate pair and encode
548
549var j = r'"\ud83e\udd26"'
550echo $j | json read (&c1)
551
552json write (c1)
553
554var j = r'"\uD83E\uDD26"'
555echo $j | json read (&c2)
556
557json write (c2)
558
559# Not a surrogate pair
560var j = r'"\u0001\u0002"'
561echo $j | json read (&c3)
562
563json write (c3)
564
565var j = r'"\u0100\u0101\u0102"'
566echo $j | json read (&c4)
567
568json write (c4)
569
570## STDOUT:
571"🤦"
572"🤦"
573"\u0001\u0002"
574"ĀāĂ"
575## END
576
577#### round trip: decode surrogate half and encode
578
579shopt -s ysh:upgrade
580
581for j in '"\ud83e"' '"\udd26"' {
582 var s = fromJson(j)
583 write -- "$j"
584 pp line (s)
585
586 write -n 'json '; json write (s)
587
588 write -n 'json8 '; json8 write (s)
589
590 echo
591}
592
593## STDOUT:
594"\ud83e"
595(Str) b'\yed\ya0\ybe'
596json "\ud83e"
597json8 b'\yed\ya0\ybe'
598
599"\udd26"
600(Str) b'\yed\yb4\ya6'
601json "\udd26"
602json8 b'\yed\yb4\ya6'
603
604## END
605
606#### toJson() toJson8()
607
608var obj = [42, 1.5, null, true, "hi", b'\yf0']
609
610echo $[toJson(obj)]
611echo $[toJson8(obj)]
612
613var obj2 = [3, 4]
614echo $[toJson(obj2, space=0)] # same as the default
615echo $[toJson8(obj2, space=0)]
616
617echo $[toJson(obj2, space=2)]
618echo $[toJson8(obj2, space=2)]
619
620# fully specify this behavior
621echo $[toJson(obj2, space=-2)]
622echo $[toJson8(obj2, space=-2)]
623
624## STDOUT:
625[42,1.5,null,true,"hi","�"]
626[42,1.5,null,true,"hi",b'\yf0']
627[3,4]
628[3,4]
629[
630 3,
631 4
632]
633[
634 3,
635 4
636]
637[3,4]
638[3,4]
639## END
640
641#### fromJson() fromJson8()
642
643var m1 = '[42,1.5,null,true,"hi"]'
644
645# JSON8 message
646var m2 = '[42,1.5,null,true,"hi",' ++ "u''" ++ ']'
647
648pp line (fromJson8(m1))
649pp line (fromJson(m1))
650
651pp line (fromJson8(m2))
652pp line (fromJson(m2)) # fails
653
654## status: 4
655## STDOUT:
656(List) [42,1.5,null,true,"hi"]
657(List) [42,1.5,null,true,"hi"]
658(List) [42,1.5,null,true,"hi",""]
659## END
660
661#### User can handle errors - toJson() toJson8()
662shopt -s ysh:upgrade
663
664var obj = []
665call obj->append(obj)
666
667try {
668 echo $[toJson(obj)]
669}
670echo status=$_status
671echo "encode error $[_error.message]" | sed 's/0x[a-f0-9]\+/(object id)/'
672
673try { # use different style
674 echo $[toJson8( /d+/ )]
675}
676echo status=$_status
677echo "encode error $[_error.message]"
678
679# This makes the interpreter fail with a message
680echo $[toJson(obj)]
681
682## status: 4
683## STDOUT:
684status=4
685encode error Can't encode List (object id) in object cycle
686status=4
687encode error Can't serialize object of type Eggex
688## END
689
690#### User can handle errors - fromJson() fromJson8()
691shopt -s ysh:upgrade
692
693var message ='[42,1.5,null,true,"hi"'
694
695try {
696 var obj = fromJson(message)
697}
698echo status=$_status
699echo "decode error $[_error.message]" | egrep -o '.*Expected.*RBracket'
700
701try {
702 var obj = fromJson8(message)
703}
704echo status=$_status
705echo "decode error $[_error.message]" | egrep -o '.*Expected.*RBracket'
706
707try {
708 var obj = fromJson('[+]')
709}
710echo "positions $[_error.start_pos] - $[_error.end_pos]"
711
712# This makes the interpreter fail with a message
713var obj = fromJson(message)
714
715## status: 4
716## STDOUT:
717status=4
718decode error Expected Id.J8_RBracket
719status=4
720decode error Expected Id.J8_RBracket
721positions 1 - 2
722## END
723
724
725#### ASCII control chars can't appear literally in messages
726shopt -s ysh:upgrade
727
728var message=$'"\x01"'
729#echo $message | od -c
730
731try {
732 var obj = fromJson(message)
733}
734echo status=$_status
735echo "$[_error.message]" | egrep -o 'ASCII control chars'
736
737## STDOUT:
738status=4
739ASCII control chars
740## END
741
742
743#### \yff can't appear in u'' code strings (command)
744
745shopt -s ysh:upgrade
746
747echo -n b'\yfd' | od -A n -t x1
748echo -n u'\yfd' | od -A n -t x1
749
750## status: 2
751## STDOUT:
752 fd
753## END
754
755#### \yff can't appear in u'' code strings (expr)
756
757var x = b'\yfe'
758write -n -- $x | od -A n -t x1
759
760var x = u'\yfe'
761write -n -- $x | od -A n -t x1
762
763## status: 2
764## STDOUT:
765 fe
766## END
767
768#### \yff can't appear in u'' multiline code strings
769
770shopt -s ysh:upgrade
771
772echo -n b'''\yfc''' | od -A n -t x1
773echo -n u'''\yfd''' | od -A n -t x1
774
775## status: 2
776## STDOUT:
777 fc
778## END
779
780#### \yff can't appear in u'' data strings
781
782#shopt -s ysh:upgrade
783
784json8 read (&b) <<'EOF'
785b'\yfe'
786EOF
787pp line (b)
788
789json8 read (&u) <<'EOF'
790u'\yfe'
791EOF
792pp line (u) # undefined
793
794## status: 1
795## STDOUT:
796(Str) b'\yfe'
797## END
798
799#### \u{dc00} can't be in surrogate range in code (command)
800
801shopt -s ysh:upgrade
802
803echo -n u'\u{dc00}' | od -A n -t x1
804
805## status: 2
806## STDOUT:
807## END
808
809#### \u{dc00} can't be in surrogate range in code (expr)
810
811shopt -s ysh:upgrade
812
813var x = u'\u{dc00}'
814echo $x | od -A n -t x1
815
816## status: 2
817## STDOUT:
818## END
819
820#### \u{dc00} can't be in surrogate range in data
821
822json8 read <<'EOF'
823["long string", u'hello \u{d7ff}', "other"]
824EOF
825echo status=$?
826
827json8 read <<'EOF'
828["long string", u'hello \u{d800}', "other"]
829EOF
830echo status=$?
831
832json8 read <<'EOF'
833["long string", u'hello \u{dfff}', "other"]
834EOF
835echo status=$?
836
837json8 read <<'EOF'
838["long string", u'hello \u{e000}', "other"]
839EOF
840echo status=$?
841
842
843## STDOUT:
844status=0
845status=1
846status=1
847status=0
848## END
849
850
851
852#### Inf and NaN can't be encoded or decoded
853
854# This works in Python, should probably support it
855
856var n = float("NaN")
857var i = float("inf")
858
859pp line (n)
860pp line (i)
861
862json dump (n)
863json dump (i)
864
865## status: 2
866## STDOUT:
867## END
868
869#### Invalid UTF-8 in JSON is rejected
870
871echo $'"\xff"' | json read
872echo status=$?
873
874echo $'"\xff"' | json8 read
875echo status=$?
876
877echo $'\xff' | json read
878echo status=$?
879
880echo $'\xff' | json8 read
881echo status=$?
882
883## STDOUT:
884status=1
885status=1
886status=1
887status=1
888## END
889
890#### Invalid JSON in J8 is rejected
891
892json8 read <<EOF
893b'$(echo -e -n '\xff')'
894EOF
895echo status=$?
896
897json8 read <<EOF
898u'$(echo -e -n '\xff')'
899EOF
900echo status=$?
901
902## STDOUT:
903status=1
904status=1
905## END
906
907#### '' means the same thing as u''
908
909echo "''" | json8 read
910pp line (_reply)
911
912echo "'\u{3bc}'" | json8 read
913pp line (_reply)
914
915echo "'\yff'" | json8 read
916echo status=$?
917
918## STDOUT:
919(Str) ""
920(Str) "μ"
921status=1
922## END
923
924#### decode deeply nested structure (stack overflow)
925
926shopt -s ysh:upgrade
927
928proc pairs(n) {
929 var m = int(n) # TODO: 1 .. n should auto-convert?
930
931 for i in (1 .. m) {
932 write -n -- '['
933 }
934 for i in (1 .. m) {
935 write -n -- ']'
936 }
937}
938
939# This is all Python can handle; C++ can handle more
940msg=$(pairs 50)
941
942#echo $msg
943
944echo "$msg" | json read
945pp line (_reply)
946echo len=$[len(_reply)]
947
948## STDOUT:
949(List) [[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]
950len=1
951## END
952
953#### decode integer larger than 2^32
954
955json=$(( 1 << 33 ))
956echo $json
957
958echo $json | json read
959pp line (_reply)
960
961## STDOUT:
9628589934592
963(Int) 8589934592
964## END
965
966#### round trip: read/write with ysh
967
968var file = "$REPO_ROOT/spec/testdata/bug.json"
969#cat $file
970cat $file | json read (&cfg)
971json write (cfg) > ysh-json
972
973diff -u $file ysh-json
974echo diff=$?
975
976## STDOUT:
977diff=0
978## END
979
980#### round trip: read/write with ysh, read/write with Python 3 (bug regression)
981
982var file = "$REPO_ROOT/spec/testdata/bug.json"
983#cat $file
984cat $file | json read (&cfg)
985json write (cfg) > ysh-json
986
987cat ysh-json | python3 -c \
988 'import json, sys; obj = json.load(sys.stdin); json.dump(obj, sys.stdout, indent=2); print()' \
989 > py-json
990
991diff -u $file py-json
992echo diff=$?
993
994## STDOUT:
995diff=0
996## END
997
998#### Encoding bytes that don't hit UTF8_REJECT immediately (bug fix)
999
1000var x = $'\xce'
1001json8 write (x)
1002declare -p x
1003echo
1004
1005var y = $'\xbc'
1006json8 write (y)
1007declare -p y
1008echo
1009
1010var z = $'\xf0\x9f\xa4\xff'
1011json8 write (z)
1012declare -p z
1013
1014## STDOUT:
1015b'\yce'
1016declare -- x=$'\xce'
1017
1018b'\ybc'
1019declare -- y=$'\xbc'
1020
1021b'\yf0\y9f\ya4\yff'
1022declare -- z=$'\xf0\x9f\xa4\xff'
1023## END
1024
1025#### NIL8 token in JSON / JSON8
1026
1027echo "(" | json read
1028echo status=$?
1029
1030echo ")" | json8 read
1031echo status=$?
1032
1033## STDOUT:
1034status=1
1035status=1
1036## END