government

commit 307ad2c9a167e314a75ac46d5ee0382f494bf1cb

tree

parent:
bb4fd939071a383f99c69ff44ab41ee227afef87

nmyk <nick@nmyk.io>

2026-11-14T11:16:58-05:00

add tokenizer

diff --git a/sexpr/token.go b/sexpr/token.go
new file mode 100644
index 0000000000000000000000000000000000000000..9fca94467ee6847ebf045fae435e43607582e04b
--- /dev/null
+++ b/sexpr/token.go
@@ -0,0 +1,288 @@
+package sexpr
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+type TokenKind int
+
+const (
+	TokEOF TokenKind = iota
+	TokLParen
+	TokRParen
+	TokQuote
+	TokDot
+	TokAtom
+	TokString
+	TokComment
+)
+
+func (k TokenKind) String() string {
+	switch k {
+	case TokEOF:
+		return "EOF"
+	case TokLParen:
+		return "("
+	case TokRParen:
+		return ")"
+	case TokQuote:
+		return "'"
+	case TokDot:
+		return "."
+	case TokAtom:
+		return "atom"
+	case TokString:
+		return "string"
+	case TokComment:
+		return "comment"
+	}
+	return fmt.Sprintf("TokenKind(%d)", int(k))
+}
+
+// Pos is a 1-based source position. Col counts runes, not bytes, so it
+// matches what an editor shows.
+type Pos struct {
+	Line, Col int
+}
+
+func (p Pos) String() string { return fmt.Sprintf("%d:%d", p.Line, p.Col) }
+
+// Token.Text depends on Kind:
+//   - TokAtom: the raw atom text. Whether it is an integer, float or symbol
+//     is the reader's decision, not the lexer's.
+//   - TokString: the decoded value, escapes already applied.
+//   - TokComment: everything after the first ';' up to the end of the line,
+//     untrimmed, so ";=> 3" has Text "=> 3".
+type Token struct {
+	Kind TokenKind
+	Text string
+	Pos  Pos
+}
+
+type SyntaxError struct {
+	Pos Pos
+	Msg string
+}
+
+func (e *SyntaxError) Error() string { return e.Pos.String() + ": " + e.Msg }
+
+// Lexer splits source text into tokens. Comments are tokens rather than
+// skipped, because later steps keep them as notes in the certificates.
+type Lexer struct {
+	src string
+	off int // byte offset of the next unread rune
+	pos Pos // position of the next unread rune
+	err error
+}
+
+func NewLexer(src string) *Lexer {
+	l := &Lexer{src: src, pos: Pos{Line: 1, Col: 1}}
+	if i := invalidUTF8(src); i >= 0 {
+		l.err = &SyntaxError{Pos: positionAt(src, i), Msg: "invalid UTF-8"}
+	}
+	return l
+}
+
+// Tokenize returns every token in src, without the final TokEOF.
+func Tokenize(src string) ([]Token, error) {
+	l := NewLexer(src)
+	var toks []Token
+	for {
+		t, err := l.Next()
+		if err != nil {
+			return nil, err
+		}
+		if t.Kind == TokEOF {
+			return toks, nil
+		}
+		toks = append(toks, t)
+	}
+}
+
+// Next returns the next token, or TokEOF at the end of input. After an
+// error, every call returns the same error.
+func (l *Lexer) Next() (Token, error) {
+	if l.err != nil {
+		return Token{}, l.err
+	}
+	t, err := l.next()
+	if err != nil {
+		l.err = err
+	}
+	return t, err
+}
+
+func (l *Lexer) next() (Token, error) {
+	for l.off < len(l.src) && unicode.IsSpace(l.peek()) {
+		l.advance()
+	}
+	start := l.pos
+	if l.off >= len(l.src) {
+		return Token{Kind: TokEOF, Pos: start}, nil
+	}
+
+	switch r := l.peek(); r {
+	case '(':
+		l.advance()
+		return Token{Kind: TokLParen, Text: "(", Pos: start}, nil
+	case ')':
+		l.advance()
+		return Token{Kind: TokRParen, Text: ")", Pos: start}, nil
+	case '\'':
+		l.advance()
+		return Token{Kind: TokQuote, Text: "'", Pos: start}, nil
+	case '"':
+		return l.lexString(start)
+	case ';':
+		l.advance()
+		begin := l.off
+		for l.off < len(l.src) && l.peek() != '\n' {
+			l.advance()
+		}
+		return Token{Kind: TokComment, Text: l.src[begin:l.off], Pos: start}, nil
+	}
+
+	begin := l.off
+	for l.off < len(l.src) && !isDelimiter(l.peek()) {
+		l.advance()
+	}
+	text := l.src[begin:l.off]
+	// A lone dot is the dotted-pair marker; "3.14" and "a.b" are atoms.
+	if text == "." {
+		return Token{Kind: TokDot, Text: ".", Pos: start}, nil
+	}
+	return Token{Kind: TokAtom, Text: text, Pos: start}, nil
+}
+
+func isDelimiter(r rune) bool {
+	switch r {
+	case '(', ')', '\'', '"', ';':
+		return true
+	}
+	return unicode.IsSpace(r)
+}
+
+// lexString reads a string literal starting at its opening quote. Strings
+// may span lines. Errors point at the offending escape, or at the opening
+// quote when the string never ends.
+func (l *Lexer) lexString(start Pos) (Token, error) {
+	l.advance()
+	var b strings.Builder
+	for {
+		if l.off >= len(l.src) {
+			return Token{}, &SyntaxError{Pos: start, Msg: "unterminated string"}
+		}
+		escPos := l.pos
+		r := l.advance()
+		switch r {
+		case '"':
+			return Token{Kind: TokString, Text: b.String(), Pos: start}, nil
+		case '\\':
+			decoded, err := l.lexEscape(escPos)
+			if err != nil {
+				return Token{}, err
+			}
+			b.WriteRune(decoded)
+		default:
+			b.WriteRune(r)
+		}
+	}
+}
+
+func (l *Lexer) lexEscape(escPos Pos) (rune, error) {
+	if l.off >= len(l.src) {
+		return 0, &SyntaxError{Pos: escPos, Msg: "unterminated escape"}
+	}
+	switch r := l.advance(); r {
+	case '"', '\\':
+		return r, nil
+	case 'n':
+		return '\n', nil
+	case 't':
+		return '\t', nil
+	case 'u':
+		return l.lexUnicodeEscape(escPos)
+	default:
+		return 0, &SyntaxError{Pos: escPos, Msg: fmt.Sprintf("unknown escape \\%c", r)}
+	}
+}
+
+// lexUnicodeEscape reads the {hex} part of \u{hex}: 1 to 6 hex digits naming
+// a Unicode scalar value (a code point that is not a surrogate).
+func (l *Lexer) lexUnicodeEscape(escPos Pos) (rune, error) {
+	bad := func(msg string) (rune, error) {
+		return 0, &SyntaxError{Pos: escPos, Msg: msg}
+	}
+	if l.off >= len(l.src) || l.peek() != '{' {
+		return bad(`\u must be followed by {hex}`)
+	}
+	l.advance()
+	begin := l.off
+	for l.off < len(l.src) && l.peek() != '}' && l.peek() != '"' {
+		l.advance()
+	}
+	if l.off >= len(l.src) || l.peek() != '}' {
+		return bad(`unterminated \u{...} escape`)
+	}
+	digits := l.src[begin:l.off]
+	l.advance()
+	if len(digits) == 0 || len(digits) > 6 {
+		return bad(`\u{...} needs 1 to 6 hex digits`)
+	}
+	n, err := strconv.ParseUint(digits, 16, 32)
+	if err != nil {
+		return bad(fmt.Sprintf(`invalid hex digits in \u{%s}`, digits))
+	}
+	r := rune(n)
+	if r > unicode.MaxRune || (r >= 0xD800 && r <= 0xDFFF) {
+		return bad(fmt.Sprintf(`\u{%s} is not a Unicode scalar value`, digits))
+	}
+	return r, nil
+}
+
+func (l *Lexer) peek() rune {
+	r, _ := utf8.DecodeRuneInString(l.src[l.off:])
+	return r
+}
+
+func (l *Lexer) advance() rune {
+	r, size := utf8.DecodeRuneInString(l.src[l.off:])
+	l.off += size
+	if r == '\n' {
+		l.pos.Line++
+		l.pos.Col = 1
+	} else {
+		l.pos.Col++
+	}
+	return r
+}
+
+// invalidUTF8 returns the byte offset of the first invalid UTF-8 sequence
+// in s, or -1. Checking up front keeps the lexer free of RuneError cases.
+func invalidUTF8(s string) int {
+	for i, r := range s {
+		if r == utf8.RuneError {
+			if _, size := utf8.DecodeRuneInString(s[i:]); size == 1 {
+				return i
+			}
+		}
+	}
+	return -1
+}
+
+func positionAt(s string, off int) Pos {
+	p := Pos{Line: 1, Col: 1}
+	for _, r := range s[:off] {
+		if r == '\n' {
+			p.Line++
+			p.Col = 1
+		} else {
+			p.Col++
+		}
+	}
+	return p
+}
diff --git a/sexpr/token_test.go b/sexpr/token_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..8c71c8246e23d8e1c05940daff14f1c4ea92f363
--- /dev/null
+++ b/sexpr/token_test.go
@@ -0,0 +1,142 @@
+package sexpr
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+type tok struct {
+	kind TokenKind
+	text string
+	line int
+	col  int
+}
+
+func lexAll(t *testing.T, src string) []tok {
+	t.Helper()
+	toks, err := Tokenize(src)
+	if err != nil {
+		t.Fatalf("Tokenize(%q): %v", src, err)
+	}
+	var got []tok
+	for _, x := range toks {
+		got = append(got, tok{x.Kind, x.Text, x.Pos.Line, x.Pos.Col})
+	}
+	return got
+}
+
+func TestTokenize(t *testing.T) {
+	tests := []struct {
+		name string
+		src  string
+		want []tok
+	}{
+		{"empty", "", nil},
+		{"whitespace only", " \t\r\n ", nil},
+		{"nested parens", "(a (b))", []tok{
+			{TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokLParen, "(", 1, 4},
+			{TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6}, {TokRParen, ")", 1, 7},
+		}},
+		{"delimiters end atoms", "(+ 1 2)", []tok{
+			{TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
+			{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
+		}},
+		{"no space needed around parens", "(a)(b)", []tok{
+			{TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokRParen, ")", 1, 3},
+			{TokLParen, "(", 1, 4}, {TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6},
+		}},
+		{"quote", "'x '(1)", []tok{
+			{TokQuote, "'", 1, 1}, {TokAtom, "x", 1, 2},
+			{TokQuote, "'", 1, 4}, {TokLParen, "(", 1, 5}, {TokAtom, "1", 1, 6}, {TokRParen, ")", 1, 7},
+		}},
+		{"lone dot vs dotted atoms", "(1 . 2) 3.14 a.b .5", []tok{
+			{TokLParen, "(", 1, 1}, {TokAtom, "1", 1, 2}, {TokDot, ".", 1, 4},
+			{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
+			{TokAtom, "3.14", 1, 9}, {TokAtom, "a.b", 1, 14}, {TokAtom, ".5", 1, 18},
+		}},
+		{"operator atoms", "<= != str? -42", []tok{
+			{TokAtom, "<=", 1, 1}, {TokAtom, "!=", 1, 4}, {TokAtom, "str?", 1, 7}, {TokAtom, "-42", 1, 12},
+		}},
+		{"string ends an atom", `a"b"`, []tok{
+			{TokAtom, "a", 1, 1}, {TokString, "b", 1, 2},
+		}},
+		{"escapes are decoded", `"a\"b\\c\nd\te"`, []tok{
+			{TokString, "a\"b\\c\nd\te", 1, 1},
+		}},
+		{"unicode escapes", `"\u{41}\u{e9}\u{1F600}"`, []tok{
+			{TokString, "Aé😀", 1, 1},
+		}},
+		{"empty string", `""`, []tok{{TokString, "", 1, 1}}},
+		{"multi-line string", "\"a\nb\" c", []tok{
+			{TokString, "a\nb", 1, 1}, {TokAtom, "c", 2, 4},
+		}},
+		{"columns count runes", "(héllo 😀 x)", []tok{
+			{TokLParen, "(", 1, 1}, {TokAtom, "héllo", 1, 2}, {TokAtom, "😀", 1, 8},
+			{TokAtom, "x", 1, 10}, {TokRParen, ")", 1, 11},
+		}},
+		{"comments keep text after first semicolon", "(+ 1 2) ;=> 3\n;; note\nx", []tok{
+			{TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
+			{TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
+			{TokComment, "=> 3", 1, 9}, {TokComment, "; note", 2, 1}, {TokAtom, "x", 3, 1},
+		}},
+		{"comment ends an atom", "abc;rest", []tok{
+			{TokAtom, "abc", 1, 1}, {TokComment, "rest", 1, 4},
+		}},
+		{"comment at end of input", "x ;", []tok{
+			{TokAtom, "x", 1, 1}, {TokComment, "", 1, 3},
+		}},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := lexAll(t, tt.src)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Tokenize(%q)\n got  %v\n want %v", tt.src, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestTokenizeErrors(t *testing.T) {
+	tests := []struct {
+		name string
+		src  string
+		want string
+	}{
+		{"unterminated string points at opening quote", "(a\n  \"abc", `2:3: unterminated string`},
+		{"unterminated after escape", `"abc\`, `1:5: unterminated escape`},
+		{"unknown escape", `"a\qb"`, `1:3: unknown escape \q`},
+		{"unicode escape without braces", `"\u41"`, `1:2: \u must be followed by {hex}`},
+		{"unicode escape unterminated", `"\u{41"`, `1:2: unterminated \u{...} escape`},
+		{"unicode escape empty", `"\u{}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
+		{"unicode escape too long", `"\u{0000041}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
+		{"unicode escape not hex", `"\u{zz}"`, `1:2: invalid hex digits in \u{zz}`},
+		{"surrogate", `"\u{D800}"`, `1:2: \u{D800} is not a Unicode scalar value`},
+		{"beyond max rune", `"\u{110000}"`, `1:2: \u{110000} is not a Unicode scalar value`},
+		{"invalid utf-8", "ab\n c\xffd", `2:3: invalid UTF-8`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := Tokenize(tt.src)
+			var se *SyntaxError
+			if !errors.As(err, &se) {
+				t.Fatalf("Tokenize(%q) error = %v, want SyntaxError", tt.src, err)
+			}
+			if err.Error() != tt.want {
+				t.Errorf("Tokenize(%q) error = %q, want %q", tt.src, err.Error(), tt.want)
+			}
+		})
+	}
+}
+
+func TestLexerErrorIsSticky(t *testing.T) {
+	l := NewLexer(`a "oops`)
+	if tk, err := l.Next(); err != nil || tk.Text != "a" {
+		t.Fatalf("first Next() = %v, %v", tk, err)
+	}
+	_, err1 := l.Next()
+	_, err2 := l.Next()
+	if err1 == nil || err1 != err2 {
+		t.Errorf("errors = %v, %v; want the same non-nil error twice", err1, err2)
+	}
+}