parent:
bb4fd939071a383f99c69ff44ab41ee227afef87
nmyk <nick@nmyk.io>
2026-11-14T11:16:58-05:00
add tokenizer
diff --git a/sexpr/token.go b/sexpr/token.go
new file mode 100644
index 0000000000000000000000000000000000000000..9fca94467ee6847ebf045fae435e43607582e04b
--- /dev/null
+++ b/sexpr/token.go
@@ -0,0 +1,288 @@
+package sexpr
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+ "unicode"
+ "unicode/utf8"
+)
+
+type TokenKind int
+
+const (
+ TokEOF TokenKind = iota
+ TokLParen
+ TokRParen
+ TokQuote
+ TokDot
+ TokAtom
+ TokString
+ TokComment
+)
+
+func (k TokenKind) String() string {
+ switch k {
+ case TokEOF:
+ return "EOF"
+ case TokLParen:
+ return "("
+ case TokRParen:
+ return ")"
+ case TokQuote:
+ return "'"
+ case TokDot:
+ return "."
+ case TokAtom:
+ return "atom"
+ case TokString:
+ return "string"
+ case TokComment:
+ return "comment"
+ }
+ return fmt.Sprintf("TokenKind(%d)", int(k))
+}
+
+// Pos is a 1-based source position. Col counts runes, not bytes, so it
+// matches what an editor shows.
+type Pos struct {
+ Line, Col int
+}
+
+func (p Pos) String() string { return fmt.Sprintf("%d:%d", p.Line, p.Col) }
+
+// Token.Text depends on Kind:
+// - TokAtom: the raw atom text. Whether it is an integer, float or symbol
+// is the reader's decision, not the lexer's.
+// - TokString: the decoded value, escapes already applied.
+// - TokComment: everything after the first ';' up to the end of the line,
+// untrimmed, so ";=> 3" has Text "=> 3".
+type Token struct {
+ Kind TokenKind
+ Text string
+ Pos Pos
+}
+
+type SyntaxError struct {
+ Pos Pos
+ Msg string
+}
+
+func (e *SyntaxError) Error() string { return e.Pos.String() + ": " + e.Msg }
+
+// Lexer splits source text into tokens. Comments are tokens rather than
+// skipped, because later steps keep them as notes in the certificates.
+type Lexer struct {
+ src string
+ off int // byte offset of the next unread rune
+ pos Pos // position of the next unread rune
+ err error
+}
+
+func NewLexer(src string) *Lexer {
+ l := &Lexer{src: src, pos: Pos{Line: 1, Col: 1}}
+ if i := invalidUTF8(src); i >= 0 {
+ l.err = &SyntaxError{Pos: positionAt(src, i), Msg: "invalid UTF-8"}
+ }
+ return l
+}
+
+// Tokenize returns every token in src, without the final TokEOF.
+func Tokenize(src string) ([]Token, error) {
+ l := NewLexer(src)
+ var toks []Token
+ for {
+ t, err := l.Next()
+ if err != nil {
+ return nil, err
+ }
+ if t.Kind == TokEOF {
+ return toks, nil
+ }
+ toks = append(toks, t)
+ }
+}
+
+// Next returns the next token, or TokEOF at the end of input. After an
+// error, every call returns the same error.
+func (l *Lexer) Next() (Token, error) {
+ if l.err != nil {
+ return Token{}, l.err
+ }
+ t, err := l.next()
+ if err != nil {
+ l.err = err
+ }
+ return t, err
+}
+
+func (l *Lexer) next() (Token, error) {
+ for l.off < len(l.src) && unicode.IsSpace(l.peek()) {
+ l.advance()
+ }
+ start := l.pos
+ if l.off >= len(l.src) {
+ return Token{Kind: TokEOF, Pos: start}, nil
+ }
+
+ switch r := l.peek(); r {
+ case '(':
+ l.advance()
+ return Token{Kind: TokLParen, Text: "(", Pos: start}, nil
+ case ')':
+ l.advance()
+ return Token{Kind: TokRParen, Text: ")", Pos: start}, nil
+ case '\'':
+ l.advance()
+ return Token{Kind: TokQuote, Text: "'", Pos: start}, nil
+ case '"':
+ return l.lexString(start)
+ case ';':
+ l.advance()
+ begin := l.off
+ for l.off < len(l.src) && l.peek() != '\n' {
+ l.advance()
+ }
+ return Token{Kind: TokComment, Text: l.src[begin:l.off], Pos: start}, nil
+ }
+
+ begin := l.off
+ for l.off < len(l.src) && !isDelimiter(l.peek()) {
+ l.advance()
+ }
+ text := l.src[begin:l.off]
+ // A lone dot is the dotted-pair marker; "3.14" and "a.b" are atoms.
+ if text == "." {
+ return Token{Kind: TokDot, Text: ".", Pos: start}, nil
+ }
+ return Token{Kind: TokAtom, Text: text, Pos: start}, nil
+}
+
+func isDelimiter(r rune) bool {
+ switch r {
+ case '(', ')', '\'', '"', ';':
+ return true
+ }
+ return unicode.IsSpace(r)
+}
+
+// lexString reads a string literal starting at its opening quote. Strings
+// may span lines. Errors point at the offending escape, or at the opening
+// quote when the string never ends.
+func (l *Lexer) lexString(start Pos) (Token, error) {
+ l.advance()
+ var b strings.Builder
+ for {
+ if l.off >= len(l.src) {
+ return Token{}, &SyntaxError{Pos: start, Msg: "unterminated string"}
+ }
+ escPos := l.pos
+ r := l.advance()
+ switch r {
+ case '"':
+ return Token{Kind: TokString, Text: b.String(), Pos: start}, nil
+ case '\\':
+ decoded, err := l.lexEscape(escPos)
+ if err != nil {
+ return Token{}, err
+ }
+ b.WriteRune(decoded)
+ default:
+ b.WriteRune(r)
+ }
+ }
+}
+
+func (l *Lexer) lexEscape(escPos Pos) (rune, error) {
+ if l.off >= len(l.src) {
+ return 0, &SyntaxError{Pos: escPos, Msg: "unterminated escape"}
+ }
+ switch r := l.advance(); r {
+ case '"', '\\':
+ return r, nil
+ case 'n':
+ return '\n', nil
+ case 't':
+ return '\t', nil
+ case 'u':
+ return l.lexUnicodeEscape(escPos)
+ default:
+ return 0, &SyntaxError{Pos: escPos, Msg: fmt.Sprintf("unknown escape \\%c", r)}
+ }
+}
+
+// lexUnicodeEscape reads the {hex} part of \u{hex}: 1 to 6 hex digits naming
+// a Unicode scalar value (a code point that is not a surrogate).
+func (l *Lexer) lexUnicodeEscape(escPos Pos) (rune, error) {
+ bad := func(msg string) (rune, error) {
+ return 0, &SyntaxError{Pos: escPos, Msg: msg}
+ }
+ if l.off >= len(l.src) || l.peek() != '{' {
+ return bad(`\u must be followed by {hex}`)
+ }
+ l.advance()
+ begin := l.off
+ for l.off < len(l.src) && l.peek() != '}' && l.peek() != '"' {
+ l.advance()
+ }
+ if l.off >= len(l.src) || l.peek() != '}' {
+ return bad(`unterminated \u{...} escape`)
+ }
+ digits := l.src[begin:l.off]
+ l.advance()
+ if len(digits) == 0 || len(digits) > 6 {
+ return bad(`\u{...} needs 1 to 6 hex digits`)
+ }
+ n, err := strconv.ParseUint(digits, 16, 32)
+ if err != nil {
+ return bad(fmt.Sprintf(`invalid hex digits in \u{%s}`, digits))
+ }
+ r := rune(n)
+ if r > unicode.MaxRune || (r >= 0xD800 && r <= 0xDFFF) {
+ return bad(fmt.Sprintf(`\u{%s} is not a Unicode scalar value`, digits))
+ }
+ return r, nil
+}
+
+func (l *Lexer) peek() rune {
+ r, _ := utf8.DecodeRuneInString(l.src[l.off:])
+ return r
+}
+
+func (l *Lexer) advance() rune {
+ r, size := utf8.DecodeRuneInString(l.src[l.off:])
+ l.off += size
+ if r == '\n' {
+ l.pos.Line++
+ l.pos.Col = 1
+ } else {
+ l.pos.Col++
+ }
+ return r
+}
+
+// invalidUTF8 returns the byte offset of the first invalid UTF-8 sequence
+// in s, or -1. Checking up front keeps the lexer free of RuneError cases.
+func invalidUTF8(s string) int {
+ for i, r := range s {
+ if r == utf8.RuneError {
+ if _, size := utf8.DecodeRuneInString(s[i:]); size == 1 {
+ return i
+ }
+ }
+ }
+ return -1
+}
+
+func positionAt(s string, off int) Pos {
+ p := Pos{Line: 1, Col: 1}
+ for _, r := range s[:off] {
+ if r == '\n' {
+ p.Line++
+ p.Col = 1
+ } else {
+ p.Col++
+ }
+ }
+ return p
+}
diff --git a/sexpr/token_test.go b/sexpr/token_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..8c71c8246e23d8e1c05940daff14f1c4ea92f363
--- /dev/null
+++ b/sexpr/token_test.go
@@ -0,0 +1,142 @@
+package sexpr
+
+import (
+ "errors"
+ "reflect"
+ "testing"
+)
+
+type tok struct {
+ kind TokenKind
+ text string
+ line int
+ col int
+}
+
+func lexAll(t *testing.T, src string) []tok {
+ t.Helper()
+ toks, err := Tokenize(src)
+ if err != nil {
+ t.Fatalf("Tokenize(%q): %v", src, err)
+ }
+ var got []tok
+ for _, x := range toks {
+ got = append(got, tok{x.Kind, x.Text, x.Pos.Line, x.Pos.Col})
+ }
+ return got
+}
+
+func TestTokenize(t *testing.T) {
+ tests := []struct {
+ name string
+ src string
+ want []tok
+ }{
+ {"empty", "", nil},
+ {"whitespace only", " \t\r\n ", nil},
+ {"nested parens", "(a (b))", []tok{
+ {TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokLParen, "(", 1, 4},
+ {TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6}, {TokRParen, ")", 1, 7},
+ }},
+ {"delimiters end atoms", "(+ 1 2)", []tok{
+ {TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
+ {TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
+ }},
+ {"no space needed around parens", "(a)(b)", []tok{
+ {TokLParen, "(", 1, 1}, {TokAtom, "a", 1, 2}, {TokRParen, ")", 1, 3},
+ {TokLParen, "(", 1, 4}, {TokAtom, "b", 1, 5}, {TokRParen, ")", 1, 6},
+ }},
+ {"quote", "'x '(1)", []tok{
+ {TokQuote, "'", 1, 1}, {TokAtom, "x", 1, 2},
+ {TokQuote, "'", 1, 4}, {TokLParen, "(", 1, 5}, {TokAtom, "1", 1, 6}, {TokRParen, ")", 1, 7},
+ }},
+ {"lone dot vs dotted atoms", "(1 . 2) 3.14 a.b .5", []tok{
+ {TokLParen, "(", 1, 1}, {TokAtom, "1", 1, 2}, {TokDot, ".", 1, 4},
+ {TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
+ {TokAtom, "3.14", 1, 9}, {TokAtom, "a.b", 1, 14}, {TokAtom, ".5", 1, 18},
+ }},
+ {"operator atoms", "<= != str? -42", []tok{
+ {TokAtom, "<=", 1, 1}, {TokAtom, "!=", 1, 4}, {TokAtom, "str?", 1, 7}, {TokAtom, "-42", 1, 12},
+ }},
+ {"string ends an atom", `a"b"`, []tok{
+ {TokAtom, "a", 1, 1}, {TokString, "b", 1, 2},
+ }},
+ {"escapes are decoded", `"a\"b\\c\nd\te"`, []tok{
+ {TokString, "a\"b\\c\nd\te", 1, 1},
+ }},
+ {"unicode escapes", `"\u{41}\u{e9}\u{1F600}"`, []tok{
+ {TokString, "Aé😀", 1, 1},
+ }},
+ {"empty string", `""`, []tok{{TokString, "", 1, 1}}},
+ {"multi-line string", "\"a\nb\" c", []tok{
+ {TokString, "a\nb", 1, 1}, {TokAtom, "c", 2, 4},
+ }},
+ {"columns count runes", "(héllo 😀 x)", []tok{
+ {TokLParen, "(", 1, 1}, {TokAtom, "héllo", 1, 2}, {TokAtom, "😀", 1, 8},
+ {TokAtom, "x", 1, 10}, {TokRParen, ")", 1, 11},
+ }},
+ {"comments keep text after first semicolon", "(+ 1 2) ;=> 3\n;; note\nx", []tok{
+ {TokLParen, "(", 1, 1}, {TokAtom, "+", 1, 2}, {TokAtom, "1", 1, 4},
+ {TokAtom, "2", 1, 6}, {TokRParen, ")", 1, 7},
+ {TokComment, "=> 3", 1, 9}, {TokComment, "; note", 2, 1}, {TokAtom, "x", 3, 1},
+ }},
+ {"comment ends an atom", "abc;rest", []tok{
+ {TokAtom, "abc", 1, 1}, {TokComment, "rest", 1, 4},
+ }},
+ {"comment at end of input", "x ;", []tok{
+ {TokAtom, "x", 1, 1}, {TokComment, "", 1, 3},
+ }},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := lexAll(t, tt.src)
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("Tokenize(%q)\n got %v\n want %v", tt.src, got, tt.want)
+ }
+ })
+ }
+}
+
+func TestTokenizeErrors(t *testing.T) {
+ tests := []struct {
+ name string
+ src string
+ want string
+ }{
+ {"unterminated string points at opening quote", "(a\n \"abc", `2:3: unterminated string`},
+ {"unterminated after escape", `"abc\`, `1:5: unterminated escape`},
+ {"unknown escape", `"a\qb"`, `1:3: unknown escape \q`},
+ {"unicode escape without braces", `"\u41"`, `1:2: \u must be followed by {hex}`},
+ {"unicode escape unterminated", `"\u{41"`, `1:2: unterminated \u{...} escape`},
+ {"unicode escape empty", `"\u{}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
+ {"unicode escape too long", `"\u{0000041}"`, `1:2: \u{...} needs 1 to 6 hex digits`},
+ {"unicode escape not hex", `"\u{zz}"`, `1:2: invalid hex digits in \u{zz}`},
+ {"surrogate", `"\u{D800}"`, `1:2: \u{D800} is not a Unicode scalar value`},
+ {"beyond max rune", `"\u{110000}"`, `1:2: \u{110000} is not a Unicode scalar value`},
+ {"invalid utf-8", "ab\n c\xffd", `2:3: invalid UTF-8`},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := Tokenize(tt.src)
+ var se *SyntaxError
+ if !errors.As(err, &se) {
+ t.Fatalf("Tokenize(%q) error = %v, want SyntaxError", tt.src, err)
+ }
+ if err.Error() != tt.want {
+ t.Errorf("Tokenize(%q) error = %q, want %q", tt.src, err.Error(), tt.want)
+ }
+ })
+ }
+}
+
+func TestLexerErrorIsSticky(t *testing.T) {
+ l := NewLexer(`a "oops`)
+ if tk, err := l.Next(); err != nil || tk.Text != "a" {
+ t.Fatalf("first Next() = %v, %v", tk, err)
+ }
+ _, err1 := l.Next()
+ _, err2 := l.Next()
+ if err1 == nil || err1 != err2 {
+ t.Errorf("errors = %v, %v; want the same non-nil error twice", err1, err2)
+ }
+}