government / sexpr / token_test.go

@ 60772087cdbbe3e7543eaf73a6f0ffc9a26570c7 | history


package sexpr

import (
	"errors"
	"reflect"
	"testing"
)

type tok struct {
	kind TokenKind
	text string
	line int
	col  int
}

func lexAll(t *testing.T, src string) []tok {
	t.Helper()
	toks, err := Tokenize(src)
	if err != nil {
		t.Fatalf("Tokenize(%q): %v", src, err)
	}
	var got []tok
	for _, x := range toks {
		got = append(got, tok{x.Kind, x.Text, x.Pos.Line, x.Pos.Col})
	}
	return got
}

func TestTokenize(t *testing.T) {
	tests := []struct {
		name string
		src  string
		want []tok
	}{
		{"empty", "", nil},
		{"whitespace only", " \t\r\n ", nil},
		{"nested parens", "(a (b))", []tok{
			{TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokLParen, "(", 1, 4},
			{TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6}, {TokRParen, ")", 1, 7},
		}},
		{"delimiters end atoms", "(+ 1 2)", []tok{
			{TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
			{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
		}},
		{"no space needed around parens", "(a)(b)", []tok{
			{TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokRParen, ")", 1, 3},
			{TokLParen, "(", 1, 4}, {TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6},
		}},
		{"quote", "'x '(1)", []tok{
			{TokQuote, "'", 1, 1}, {TokAtom, "x", 1, 2},
			{TokQuote, "'", 1, 4}, {TokLParen, "(", 1, 5}, {TokAtom, "1", 1, 6}, {TokRParen, ")", 1, 7},
		}},
		{"lone dot vs dotted atoms", "(1 . 2) 3.14 a.b .5", []tok{
			{TokLParen, "(", 1, 1}, {TokAtom, "1", 1, 2}, {TokDot, ".", 1, 4},
			{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
			{TokAtom, "3.14", 1, 9}, {TokAtom, "a.b", 1, 14}, {TokAtom, ".5", 1, 18},
		}},
		{"operator atoms", "<= != str? -42", []tok{
			{TokAtom, "<=", 1, 1}, {TokAtom, "!=", 1, 4}, {TokAtom, "str?", 1, 7}, {TokAtom, "-42", 1, 12},
		}},
		{"string ends an atom", `a"b"`, []tok{
			{TokAtom, "a", 1, 1}, {TokString, "b", 1, 2},
		}},
		{"escapes are decoded", `"a\"b\\c\nd\te"`, []tok{
			{TokString, "a\"b\\c\nd\te", 1, 1},
		}},
		{"unicode escapes", `"\u{41}\u{e9}\u{1F600}"`, []tok{
			{TokString, "Aé😀", 1, 1},
		}},
		{"empty string", `""`, []tok{{TokString, "", 1, 1}}},
		{"multi-line string", "\"a\nb\" c", []tok{
			{TokString, "a\nb", 1, 1}, {TokAtom, "c", 2, 4},
		}},
		{"columns count runes", "(héllo 😀 x)", []tok{
			{TokLParen, "(", 1, 1}, {TokAtom, "héllo", 1, 2}, {TokAtom, "😀", 1, 8},
			{TokAtom, "x", 1, 10}, {TokRParen, ")", 1, 11},
		}},
		{"comments keep text after first semicolon", "(+ 1 2) ;=> 3\n;; note\nx", []tok{
			{TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
			{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
			{TokComment, "=> 3", 1, 9}, {TokComment, "; note", 2, 1}, {TokAtom, "x", 3, 1},
		}},
		{"comment ends an atom", "abc;rest", []tok{
			{TokAtom, "abc", 1, 1}, {TokComment, "rest", 1, 4},
		}},
		{"comment at end of input", "x ;", []tok{
			{TokAtom, "x", 1, 1}, {TokComment, "", 1, 3},
		}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			got := lexAll(t, tt.src)
			if !reflect.DeepEqual(got, tt.want) {
				t.Errorf("Tokenize(%q)\n got  %v\n want %v", tt.src, got, tt.want)
			}
		})
	}
}

func TestTokenizeErrors(t *testing.T) {
	tests := []struct {
		name string
		src  string
		want string
	}{
		{"unterminated string points at opening quote", "(a\n  \"abc", `2:3: unterminated string`},
		{"unterminated after escape", `"abc\`, `1:5: unterminated escape`},
		{"unknown escape", `"a\qb"`, `1:3: unknown escape \q`},
		{"unicode escape without braces", `"\u41"`, `1:2: \u must be followed by {hex}`},
		{"unicode escape unterminated", `"\u{41"`, `1:2: unterminated \u{...} escape`},
		{"unicode escape empty", `"\u{}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
		{"unicode escape too long", `"\u{0000041}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
		{"unicode escape not hex", `"\u{zz}"`, `1:2: invalid hex digits in \u{zz}`},
		{"surrogate", `"\u{D800}"`, `1:2: \u{D800} is not a Unicode scalar value`},
		{"beyond max rune", `"\u{110000}"`, `1:2: \u{110000} is not a Unicode scalar value`},
		{"invalid utf-8", "ab\n c\xffd", `2:3: invalid UTF-8`},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			_, err := Tokenize(tt.src)
			var se *SyntaxError
			if !errors.As(err, &se) {
				t.Fatalf("Tokenize(%q) error = %v, want SyntaxError", tt.src, err)
			}
			if err.Error() != tt.want {
				t.Errorf("Tokenize(%q) error = %q, want %q", tt.src, err.Error(), tt.want)
			}
		})
	}
}

func TestLexerErrorIsSticky(t *testing.T) {
	l := NewLexer(`a "oops`)
	if tk, err := l.Next(); err != nil || tk.Text != "a" {
		t.Fatalf("first Next() = %v, %v", tk, err)
	}
	_, err1 := l.Next()
	_, err2 := l.Next()
	if err1 == nil || err1 != err2 {
		t.Errorf("errors = %v, %v; want the same non-nil error twice", err1, err2)
	}
}