@ 26a72779eff9f25b53bae86607e9e4f6cd54934d | history
package sexpr
import (
"errors"
"reflect"
"testing"
)
type tok struct {
kind TokenKind
text string
line int
col int
}
func lexAll(t *testing.T, src string) []tok {
t.Helper()
toks, err := Tokenize(src)
if err != nil {
t.Fatalf("Tokenize(%q): %v", src, err)
}
var got []tok
for _, x := range toks {
got = append(got, tok{x.Kind, x.Text, x.Pos.Line, x.Pos.Col})
}
return got
}
func TestTokenize(t *testing.T) {
tests := []struct {
name string
src string
want []tok
}{
{"empty", "", nil},
{"whitespace only", " \t\r\n ", nil},
{"nested parens", "(a (b))", []tok{
{TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokLParen, "(", 1, 4},
{TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6}, {TokRParen, ")", 1, 7},
}},
{"delimiters end atoms", "(+ 1 2)", []tok{
{TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
}},
{"no space needed around parens", "(a)(b)", []tok{
{TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokRParen, ")", 1, 3},
{TokLParen, "(", 1, 4}, {TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6},
}},
{"quote", "'x '(1)", []tok{
{TokQuote, "'", 1, 1}, {TokAtom, "x", 1, 2},
{TokQuote, "'", 1, 4}, {TokLParen, "(", 1, 5}, {TokAtom, "1", 1, 6}, {TokRParen, ")", 1, 7},
}},
{"lone dot vs dotted atoms", "(1 . 2) 3.14 a.b .5", []tok{
{TokLParen, "(", 1, 1}, {TokAtom, "1", 1, 2}, {TokDot, ".", 1, 4},
{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
{TokAtom, "3.14", 1, 9}, {TokAtom, "a.b", 1, 14}, {TokAtom, ".5", 1, 18},
}},
{"operator atoms", "<= != str? -42", []tok{
{TokAtom, "<=", 1, 1}, {TokAtom, "!=", 1, 4}, {TokAtom, "str?", 1, 7}, {TokAtom, "-42", 1, 12},
}},
{"string ends an atom", `a"b"`, []tok{
{TokAtom, "a", 1, 1}, {TokString, "b", 1, 2},
}},
{"escapes are decoded", `"a\"b\\c\nd\te"`, []tok{
{TokString, "a\"b\\c\nd\te", 1, 1},
}},
{"unicode escapes", `"\u{41}\u{e9}\u{1F600}"`, []tok{
{TokString, "Aé😀", 1, 1},
}},
{"empty string", `""`, []tok{{TokString, "", 1, 1}}},
{"multi-line string", "\"a\nb\" c", []tok{
{TokString, "a\nb", 1, 1}, {TokAtom, "c", 2, 4},
}},
{"columns count runes", "(héllo 😀 x)", []tok{
{TokLParen, "(", 1, 1}, {TokAtom, "héllo", 1, 2}, {TokAtom, "😀", 1, 8},
{TokAtom, "x", 1, 10}, {TokRParen, ")", 1, 11},
}},
{"comments keep text after first semicolon", "(+ 1 2) ;=> 3\n;; note\nx", []tok{
{TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
{TokComment, "=> 3", 1, 9}, {TokComment, "; note", 2, 1}, {TokAtom, "x", 3, 1},
}},
{"comment ends an atom", "abc;rest", []tok{
{TokAtom, "abc", 1, 1}, {TokComment, "rest", 1, 4},
}},
{"comment at end of input", "x ;", []tok{
{TokAtom, "x", 1, 1}, {TokComment, "", 1, 3},
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := lexAll(t, tt.src)
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("Tokenize(%q)\n got %v\n want %v", tt.src, got, tt.want)
}
})
}
}
func TestTokenizeErrors(t *testing.T) {
tests := []struct {
name string
src string
want string
}{
{"unterminated string points at opening quote", "(a\n \"abc", `2:3: unterminated string`},
{"unterminated after escape", `"abc\`, `1:5: unterminated escape`},
{"unknown escape", `"a\qb"`, `1:3: unknown escape \q`},
{"unicode escape without braces", `"\u41"`, `1:2: \u must be followed by {hex}`},
{"unicode escape unterminated", `"\u{41"`, `1:2: unterminated \u{...} escape`},
{"unicode escape empty", `"\u{}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
{"unicode escape too long", `"\u{0000041}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
{"unicode escape not hex", `"\u{zz}"`, `1:2: invalid hex digits in \u{zz}`},
{"surrogate", `"\u{D800}"`, `1:2: \u{D800} is not a Unicode scalar value`},
{"beyond max rune", `"\u{110000}"`, `1:2: \u{110000} is not a Unicode scalar value`},
{"invalid utf-8", "ab\n c\xffd", `2:3: invalid UTF-8`},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := Tokenize(tt.src)
var se *SyntaxError
if !errors.As(err, &se) {
t.Fatalf("Tokenize(%q) error = %v, want SyntaxError", tt.src, err)
}
if err.Error() != tt.want {
t.Errorf("Tokenize(%q) error = %q, want %q", tt.src, err.Error(), tt.want)
}
})
}
}
func TestLexerErrorIsSticky(t *testing.T) {
l := NewLexer(`a "oops`)
if tk, err := l.Next(); err != nil || tk.Text != "a" {
t.Fatalf("first Next() = %v, %v", tk, err)
}
_, err1 := l.Next()
_, err2 := l.Next()
if err1 == nil || err1 != err2 {
t.Errorf("errors = %v, %v; want the same non-nil error twice", err1, err2)
}
}