.........8

--- echo \" \\ hi` ---

(Token
  id: Id.Backtick_Other
  col: 0
  length: 5
  span_id: 0
  line: (SourceLine line_num:1 content:'echo \\" \\\\ hi`' src:(source.MainFile path:'<lex_test.py>'))
  tval: 'echo '
)
(Token
  id: Id.Backtick_DoubleQuote
  col: 5
  length: 2
  span_id: 1
  line: (SourceLine line_num:1 content:'echo \\" \\\\ hi`' src:(source.MainFile path:'<lex_test.py>'))
  tval: '\\"'
)
(Token
  id: Id.Backtick_Other
  col: 7
  length: 1
  span_id: 2
  line: (SourceLine line_num:1 content:'echo \\" \\\\ hi`' src:(source.MainFile path:'<lex_test.py>'))
  tval: ' '
)
(Token
  id: Id.Backtick_Quoted
  col: 8
  length: 2
  span_id: 3
  line: (SourceLine line_num:1 content:'echo \\" \\\\ hi`' src:(source.MainFile path:'<lex_test.py>'))
  tval: '\\\\'
)
(Token
  id: Id.Backtick_Other
  col: 10
  length: 3
  span_id: 4
  line: (SourceLine line_num:1 content:'echo \\" \\\\ hi`' src:(source.MainFile path:'<lex_test.py>'))
  tval: ' hi'
)
(Token
  id: Id.Backtick_Right
  col: 13
  length: 1
  span_id: 5
  line: (SourceLine line_num:1 content:'echo \\" \\\\ hi`' src:(source.MainFile path:'<lex_test.py>'))
  tval: '`'
)
(Token
  id: Id.Eof_Real
  col: 14
  length: 0
  span_id: 6
  line: (SourceLine line_num:1 content:'echo \\" \\\\ hi`' src:(source.MainFile path:'<lex_test.py>'))
  tval: ''
)

--- ` ---

(Token
  id: Id.Backtick_Right
  col: 0
  length: 1
  span_id: 0
  line: (SourceLine line_num:1 content:'`' src:(source.MainFile path:'<lex_test.py>'))
  tval: '`'
)
(Token
  id: Id.Eof_Real
  col: 1
  length: 0
  span_id: 1
  line: (SourceLine line_num:1 content:'`' src:(source.MainFile path:'<lex_test.py>'))
  tval: ''
)

---  ---

(Token
  id: Id.Eof_Real
  col: 0
  length: 0
  span_id: 0
  line: (SourceLine line_num:0 content:'' src:(source.MainFile path:'<lex_test.py>'))
  tval: ''
)
(Token
  id: Id.Char_Literals
  col: 0
  length: 7
  span_id: 0
  line: 
    (SourceLine
      line_num: 1
      content: 'foo bar\\n \\x00 \\000 \\u0065'
      src: (source.MainFile path:'<lex_test.py>')
    )
  tval: 'foo bar'
)
(Token
  id: Id.Char_OneChar
  col: 7
  length: 2
  span_id: 1
  line: 
    (SourceLine
      line_num: 1
      content: 'foo bar\\n \\x00 \\000 \\u0065'
      src: (source.MainFile path:'<lex_test.py>')
    )
  tval: '\\n'
)

--- @[ ] ---

(Token
  id: Id.Expr_Reserved
  col: 0
  length: 2
  span_id: 0
  line: (SourceLine line_num:1 content:'@[ ]' src:(source.MainFile path:'<lex_test.py>'))
  tval: '@['
)
(Token
  id: Id.Ignored_Space
  col: 2
  length: 1
  span_id: 1
  line: (SourceLine line_num:1 content:'@[ ]' src:(source.MainFile path:'<lex_test.py>'))
  tval: _
)
(Token
  id: Id.Op_RBracket
  col: 3
  length: 1
  span_id: 2
  line: (SourceLine line_num:1 content:'@[ ]' src:(source.MainFile path:'<lex_test.py>'))
  tval: _
)
(Token
  id: Id.Eof_Real
  col: 4
  length: 0
  span_id: 3
  line: (SourceLine line_num:1 content:'@[ ]' src:(source.MainFile path:'<lex_test.py>'))
  tval: ''
)

--- hello %s\n ---

(Token
  id: Id.Char_Literals
  col: 0
  length: 6
  span_id: 0
  line: (SourceLine line_num:1 content:'hello %s\\n' src:(source.MainFile path:'<lex_test.py>'))
  tval: 'hello '
)
(Token
  id: Id.Format_Percent
  col: 6
  length: 1
  span_id: 1
  line: (SourceLine line_num:1 content:'hello %s\\n' src:(source.MainFile path:'<lex_test.py>'))
  tval: '%'
)
(Token
  id: Id.Char_Literals
  col: 7
  length: 1
  span_id: 2
  line: (SourceLine line_num:1 content:'hello %s\\n' src:(source.MainFile path:'<lex_test.py>'))
  tval: s
)
(Token
  id: Id.Char_OneChar
  col: 8
  length: 2
  span_id: 3
  line: (SourceLine line_num:1 content:'hello %s\\n' src:(source.MainFile path:'<lex_test.py>'))
  tval: '\\n'
)
(Token
  id: Id.Eof_Real
  col: 10
  length: 0
  span_id: 4
  line: (SourceLine line_num:1 content:'hello %s\\n' src:(source.MainFile path:'<lex_test.py>'))
  tval: ''
)

--- %% percent %%\377 ---

(Token
  id: Id.Format_EscapedPercent
  col: 0
  length: 2
  span_id: 0
  line: 
    (SourceLine
      line_num: 1
      content: '%% percent %%\\377'
      src: (source.MainFile path:'<lex_test.py>')
    )
  tval: '%%.............
----------------------------------------------------------------------
Ran 22 tests in 0.015s

OK
'
)
(Token
  id: Id.Char_Literals
  col: 2
  length: 9
  span_id: 1
  line: 
    (SourceLine
      line_num: 1
      content: '%% percent %%\\377'
      src: (source.MainFile path:'<lex_test.py>')
    )
  tval: ' percent '
)
(Token
  id: Id.Format_EscapedPercent
  col: 11
  length: 2
  span_id: 2
  line: 
    (SourceLine
      line_num: 1
      content: '%% percent %%\\377'
      src: (source.MainFile path:'<lex_test.py>')
    )
  tval: '%%'
)
(Token
  id: Id.Char_Octal3
  col: 13
  length: 4
  span_id: 3
  line: 
    (SourceLine
      line_num: 1
      content: '%% percent %%\\377'
      src: (source.MainFile path:'<lex_test.py>')
    )
  tval: '\\377'
)
(Token
  id: Id.Eof_Real
  col: 17
  length: 0
  span_id: 4
  line: 
    (SourceLine
      line_num: 1
      content: '%% percent %%\\377'
      src: (source.MainFile path:'<lex_test.py>')
    )
  tval: ''
)

--- -3.3f ---

(Token
  id: Id.Format_Flag
  col: 0
  length: 1
  span_id: 0
  line: (SourceLine line_num:1 content:-3.3f src:(source.MainFile path:'<lex_test.py>'))
  tval: -
)
(Token
  id: Id.Format_Num
  col: 1
  length: 1
  span_id: 1
  line: (SourceLine line_num:1 content:-3.3f src:(source.MainFile path:'<lex_test.py>'))
  tval: 3
)
(Token
  id: Id.Format_Dot
  col: 2
  length: 1
  span_id: 2
  line: (SourceLine line_num:1 content:-3.3f src:(source.MainFile path:'<lex_test.py>'))
  tval: .
)
(Token
  id: Id.Format_Num
  col: 3
  length: 1
  span_id: 3
  line: (SourceLine line_num:1 content:-3.3f src:(source.MainFile path:'<lex_test.py>'))
  tval: 3
)
(Token
  id: Id.Format_Type
  col: 4
  length: 1
  span_id: 4
  line: (SourceLine line_num:1 content:-3.3f src:(source.MainFile path:'<lex_test.py>'))
  tval: f
)
(Token
  id: Id.Eof_Real
  col: 5
  length: 0
  span_id: 5
  line: (SourceLine line_num:1 content:-3.3f src:(source.MainFile path:'<lex_test.py>'))
  tval: ''
)

--- 03d ---

(Token
  id: Id.Format_Zero
  col: 0
  length: 1
  span_id: 0
  line: (SourceLine line_num:1 content:03d src:(source.MainFile path:'<lex_test.py>'))
  tval: 0
)
(Token
  id: Id.Format_Num
  col: 1
  length: 1
  span_id: 1
  line: (SourceLine line_num:1 content:03d src:(source.MainFile path:'<lex_test.py>'))
  tval: 3
)
(Token
  id: Id.Format_Type
  col: 2
  length: 1
  span_id: 2
  line: (SourceLine line_num:1 content:03d src:(source.MainFile path:'<lex_test.py>'))
  tval: d
)
(Token
  id: Id.Eof_Real
  col: 3
  length: 0
  span_id: 3
  line: (SourceLine line_num:1 content:03d src:(source.MainFile path:'<lex_test.py>'))
  tval: ''
)
(Token
  id: Id.Left_SingleQuote
  col: 0
  length: 1
  span_id: 0
  line: (SourceLine line_num:1 content:'\'hi\'' src:(source.MainFile path:'<lex_test.py>'))
  tval: '\''
)
[(339, 'a'), (340, '..'), (339, 'z')]
[(338, '100'), (340, '..'), (338, '300')]
[(338, '-300'), (340, '..'), (338, '-100'), (340, '..'), (338, '1')]
[(338, '1'), (341, '.'), (338, '3')]
[(339, 'a'), (339, 'a')]
[(173, 'newline '), (162, '\\n'), (173, ' NUL '), (162, '\\0'), (173, ' octal '), (167, '\\0377'), (173, ' hex '), (164, '\\x00')]
[(173, 'unicode '), (168, '\\u0065'), (173, ' '), (170, '\\U00000065')]
[(50, '\\'), (173, 'd '), (162, '\\e'), (173, ' '), (162, '\\f'), (173, ' '), (50, '\\'), (173, 'g')]
[(96, '['), (96, '!'), (96, 'abc]')]
[(96, '${'), (96, '!'), (96, 'indirect}')]
[(96, '$'), (96, '!'), (96, 'x')]
[(96, '!'), (96, '(foo|bar)')]
[(96, 'echo hi')]
[(96, 'echo '), (93, '!!'), (96, ' '), (93, '!*'), (96, ' '), (93, '!^'), (96, ' '), (93, '!$')]
[(96, 'echo '), (96, '\\!'), (96, '!')]
[(96, 'echo '), (94, '!3'), (96, '...')]
[(96, 'echo '), (94, '!-5'), (96, '...')]
[(96, 'echo '), (95, '!x/foo.py '), (96, 'bar')]
---
[(96, 'echo '), (96, "'!!'"), (96, ' $'), (96, "'!!'"), (96, ' ')]
[(96, 'echo '), (96, "'!! ")]
[(96, 'echo '), (96, "\\'"), (96, ' '), (93, '!!'), (96, ' ')]
[(336, 'foo')]
[(332, '\\h'), (336, ' '), (332, '\\w'), (336, ' '), (332, '\\$')]
P '[^\\\\\\0]+'