parent:
307ad2c9a167e314a75ac46d5ee0382f494bf1cb
nmyk <nick@nmyk.io>
2026-11-14T11:23:05-05:00
add reader
diff --git a/sexpr/reader.go b/sexpr/reader.go
new file mode 100644
index 0000000000000000000000000000000000000000..c7edd7e2478425c515339af7003b53f7e461b02b
--- /dev/null
+++ b/sexpr/reader.go
@@ -0,0 +1,208 @@
+package sexpr
+
+import (
+ "fmt"
+ "math"
+ "regexp"
+ "strconv"
+)
+
+// Reader parses source text into values by recursive descent over the
+// grammar:
+//
+// datum = atom | string | "'" datum | "(" list
+// list = ")" | datum { datum } [ "." datum ] ")"
+//
+// Comments are skipped for now; step 7.1 keeps them as notes.
+type Reader struct {
+ lex *Lexer
+ peeked *Token
+}
+
+func NewReader(src string) *Reader {
+ return &Reader{lex: NewLexer(src)}
+}
+
+// ReadAll returns every top-level form in src.
+func ReadAll(src string) ([]Value, error) {
+ r := NewReader(src)
+ var forms []Value
+ for {
+ v, err := r.Read()
+ if err != nil {
+ return nil, err
+ }
+ if v == nil {
+ return forms, nil
+ }
+ forms = append(forms, v)
+ }
+}
+
+// Read returns the next top-level form, or nil at end of input.
+func (r *Reader) Read() (Value, error) {
+ t, err := r.peek()
+ if err != nil {
+ return nil, err
+ }
+ if t.Kind == TokEOF {
+ return nil, nil
+ }
+ return r.datum()
+}
+
+func (r *Reader) datum() (Value, error) {
+ t, err := r.next()
+ if err != nil {
+ return nil, err
+ }
+ switch t.Kind {
+ case TokAtom:
+ return parseAtom(t)
+ case TokString:
+ return String(t.Text), nil
+ case TokQuote:
+ next, err := r.peek()
+ if err != nil {
+ return nil, err
+ }
+ if next.Kind == TokEOF || next.Kind == TokRParen {
+ return nil, &SyntaxError{Pos: t.Pos, Msg: "' must be followed by a value"}
+ }
+ v, err := r.datum()
+ if err != nil {
+ return nil, err
+ }
+ return List(Symbol("quote"), v), nil
+ case TokLParen:
+ return r.list(t.Pos)
+ case TokRParen:
+ return nil, &SyntaxError{Pos: t.Pos, Msg: "unexpected )"}
+ case TokDot:
+ return nil, &SyntaxError{Pos: t.Pos, Msg: "unexpected . outside a list"}
+ }
+ panic(fmt.Sprintf("sexpr: reader got token %v", t.Kind))
+}
+
+// list parses the rest of a list after its "(". Errors about a missing ")"
+// point at the "(" that opened the list, which is where the mistake usually
+// is.
+func (r *Reader) list(open Pos) (Value, error) {
+ var items []Value
+ for {
+ t, err := r.peek()
+ if err != nil {
+ return nil, err
+ }
+ switch t.Kind {
+ case TokEOF:
+ return nil, &SyntaxError{Pos: open, Msg: "unterminated list"}
+ case TokRParen:
+ r.next()
+ return List(items...), nil
+ case TokDot:
+ if len(items) == 0 {
+ return nil, &SyntaxError{Pos: t.Pos, Msg: "a dotted pair needs a value before ."}
+ }
+ r.next()
+ return r.dottedTail(t.Pos, open, items)
+ }
+ v, err := r.datum()
+ if err != nil {
+ return nil, err
+ }
+ items = append(items, v)
+ }
+}
+
+// dottedTail parses the single value after "." and the ")" that must follow
+// it: a pair has exactly one cdr.
+func (r *Reader) dottedTail(dot, open Pos, items []Value) (Value, error) {
+ t, err := r.peek()
+ if err != nil {
+ return nil, err
+ }
+ switch t.Kind {
+ case TokRParen, TokEOF:
+ return nil, &SyntaxError{Pos: dot, Msg: "expected a value after ."}
+ case TokDot:
+ return nil, &SyntaxError{Pos: t.Pos, Msg: "only one . is allowed in a list"}
+ }
+ tail, err := r.datum()
+ if err != nil {
+ return nil, err
+ }
+ t, err = r.next()
+ if err != nil {
+ return nil, err
+ }
+ switch t.Kind {
+ case TokRParen:
+ return ListWithTail(tail, items...), nil
+ case TokEOF:
+ return nil, &SyntaxError{Pos: open, Msg: "unterminated list"}
+ case TokDot:
+ return nil, &SyntaxError{Pos: t.Pos, Msg: "only one . is allowed in a list"}
+ }
+ return nil, &SyntaxError{Pos: t.Pos, Msg: "expected ) after the value following ."}
+}
+
+// next and peek give the reader one token of lookahead, which is all this
+// grammar needs (it is LL(1)). Comments are dropped here.
+func (r *Reader) next() (Token, error) {
+ if r.peeked != nil {
+ t := *r.peeked
+ r.peeked = nil
+ return t, nil
+ }
+ for {
+ t, err := r.lex.Next()
+ if err != nil || t.Kind != TokComment {
+ return t, err
+ }
+ }
+}
+
+func (r *Reader) peek() (Token, error) {
+ if r.peeked == nil {
+ t, err := r.next()
+ if err != nil {
+ return Token{}, err
+ }
+ r.peeked = &t
+ }
+ return *r.peeked, nil
+}
+
+var (
+ intPattern = regexp.MustCompile(`^-?(0|[1-9][0-9]*)$`)
+ floatPattern = regexp.MustCompile(`^-?[0-9]+\.[0-9]+([eE][-+]?[0-9]+)?$`)
+ // Anything starting like a number must be a valid one, so "007", "1."
+ // and ".5" are errors rather than surprising symbols.
+ numberLike = regexp.MustCompile(`^[-+]?\.?[0-9]`)
+)
+
+func parseAtom(t Token) (Value, error) {
+ s := t.Text
+ switch {
+ case s == "T":
+ return T, nil
+ case s == "F":
+ return F, nil
+ case intPattern.MatchString(s):
+ n, err := strconv.ParseInt(s, 10, 64)
+ if err != nil {
+ return nil, &SyntaxError{Pos: t.Pos, Msg: fmt.Sprintf("integer %s does not fit in 64 bits", s)}
+ }
+ return Int(n), nil
+ case floatPattern.MatchString(s):
+ f, err := strconv.ParseFloat(s, 64)
+ if err != nil || math.IsInf(f, 0) {
+ return nil, &SyntaxError{Pos: t.Pos, Msg: fmt.Sprintf("float %s is out of range", s)}
+ }
+ return Float(f), nil
+ case numberLike.MatchString(s):
+ return nil, &SyntaxError{Pos: t.Pos, Msg: fmt.Sprintf("malformed number %s", s)}
+ }
+ return Symbol(s), nil
+}
diff --git a/sexpr/reader_test.go b/sexpr/reader_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..9fe4b5634a56e2bb00089fa44ff3a5fe88c1d0e5
--- /dev/null
+++ b/sexpr/reader_test.go
@@ -0,0 +1,202 @@
+package sexpr
+
+import (
+ "errors"
+ "math"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestReadAtoms(t *testing.T) {
+ tests := []struct {
+ src string
+ want Value
+ }{
+ {"0", Int(0)},
+ {"42", Int(42)},
+ {"-42", Int(-42)},
+ {"9223372036854775807", Int(math.MaxInt64)},
+ {"-9223372036854775808", Int(math.MinInt64)},
+ {"3.14", Float(3.14)},
+ {"-0.5", Float(-0.5)},
+ {"1.0", Float(1)},
+ {"1.5e3", Float(1500)},
+ {"1.0E-2", Float(0.01)},
+ {"1.0e+21", Float(1e21)},
+ {"T", T},
+ {"F", F},
+ {"t", Symbol("t")},
+ {"foo", Symbol("foo")},
+ {"+", Symbol("+")},
+ {"-", Symbol("-")},
+ {"<=", Symbol("<=")},
+ {"str?", Symbol("str?")},
+ {"-x", Symbol("-x")},
+ {"a.b", Symbol("a.b")},
+ {"x1", Symbol("x1")},
+ {"héllo", Symbol("héllo")},
+ {`"hi"`, String("hi")},
+ {`""`, String("")},
+ {`"1"`, String("1")},
+ }
+ for _, tt := range tests {
+ t.Run(tt.src, func(t *testing.T) {
+ got := readOne(t, tt.src)
+ if !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("Read(%s) = %#v, want %#v", tt.src, got, tt.want)
+ }
+ })
+ }
+}
+
+// The printed form is the normal form: reading then printing gives it.
+func TestReadNormalizes(t *testing.T) {
+ tests := []struct{ src, want string }{
+ {"()", "()"},
+ {"( )", "()"},
+ {"(1 2 3)", "(1 2 3)"},
+ {" ( 1\n\t2 3 ) ", "(1 2 3)"},
+ {"(a (b (c)) ())", "(a (b (c)) ())"},
+ {"(1 . 2)", "(1 . 2)"},
+ {"(1 2 . 3)", "(1 2 . 3)"},
+ {"(1 . (2 3))", "(1 2 3)"},
+ {"(1 . (2 . (3 . ())))", "(1 2 3)"},
+ {"(1 . ())", "(1)"},
+ {"((1 . 2) . 3)", "((1 . 2) . 3)"},
+ {"'x", "(quote x)"},
+ {"'(1 2)", "(quote (1 2))"},
+ {"''x", "(quote (quote x))"},
+ {"(a 'b c)", "(a (quote b) c)"},
+ {"'(1 . 2)", "(quote (1 . 2))"},
+ {"(1 . 'x)", "(1 quote x)"},
+ {"(+ 1 ; comment\n 2)", "(+ 1 2)"},
+ {"(define sq (lambda (x) (* x x)))", "(define sq (lambda (x) (* x x)))"},
+ {`("a\nb" "\u{41}")`, `("a\nb" "A")`},
+ {"1.50", "1.5"},
+ {"(T F)", "(T F)"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.src, func(t *testing.T) {
+ if got := Print(readOne(t, tt.src)); got != tt.want {
+ t.Errorf("Print(Read(%q)) = %s, want %s", tt.src, got, tt.want)
+ }
+ })
+ }
+}
+
+// Print's output must read back to an equal value.
+func TestPrintReadRoundTrip(t *testing.T) {
+ values := []Value{
+ Int(math.MinInt64), Int(math.MaxInt64), Float(0.1), Float(1), Float(-0.0),
+ Float(1e21), Float(1.5e-7), Float(math.MaxFloat64), Float(math.SmallestNonzeroFloat64),
+ String(""), String("a\"b\\c\nd\te"), String("bell\a\x00\x7f"), String("héllo 😀"),
+ T, F, Symbol("x"), Nil,
+ List(Int(1), List(Float(2), Nil), String("s")),
+ Cons(Int(1), Int(2)),
+ ListWithTail(Symbol("c"), Symbol("a"), Symbol("b")),
+ Cons(Cons(Int(1), Int(2)), Int(3)),
+ }
+ for _, v := range values {
+ printed := Print(v)
+ t.Run(printed, func(t *testing.T) {
+ if got := readOne(t, printed); !reflect.DeepEqual(got, v) {
+ t.Errorf("Read(Print(v)) = %s, want %s", Print(got), printed)
+ }
+ })
+ }
+}
+
+func TestReadAll(t *testing.T) {
+ forms, err := ReadAll("(define x 1) ; one\nx\n'y\n")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var got []string
+ for _, f := range forms {
+ got = append(got, Print(f))
+ }
+ want := []string{"(define x 1)", "x", "(quote y)"}
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("ReadAll = %v, want %v", got, want)
+ }
+
+ forms, err = ReadAll(" ; only a comment\n")
+ if err != nil || len(forms) != 0 {
+ t.Errorf("ReadAll(comment only) = %v, %v; want no forms", forms, err)
+ }
+}
+
+func TestReadErrors(t *testing.T) {
+ tests := []struct{ src, want string }{
+ {")", "1:1: unexpected )"},
+ {"(1 2))", "1:6: unexpected )"},
+ {"(1 2", "1:1: unterminated list"},
+ {"(a\n (b c)", "1:1: unterminated list"},
+ {"(a (b", "1:4: unterminated list"},
+ {".", "1:1: unexpected . outside a list"},
+ {"'", "1:1: ' must be followed by a value"},
+ {"(')", "1:2: ' must be followed by a value"},
+ {"'.", "1:2: unexpected . outside a list"},
+
+ // A pair has exactly one cdr.
+ {"(. 1)", "1:2: a dotted pair needs a value before ."},
+ {"(1 .)", "1:4: expected a value after ."},
+ {"(1 . )", "1:4: expected a value after ."},
+ {"(1 .", "1:4: expected a value after ."},
+ {"(1 . 2 3)", "1:8: expected ) after the value following ."},
+ {"(1 . 2 . 3)", "1:8: only one . is allowed in a list"},
+ {"(1 . . 2)", "1:6: only one . is allowed in a list"},
+ {"(1 . 2", "1:1: unterminated list"},
+
+ // Numbers must match the grammar exactly.
+ {"007", "1:1: malformed number 007"},
+ {"1.", "1:1: malformed number 1."},
+ {".5", "1:1: malformed number .5"},
+ {"-.5", "1:1: malformed number -.5"},
+ {"+5", "1:1: malformed number +5"},
+ {"1e5", "1:1: malformed number 1e5"},
+ {"12abc", "1:1: malformed number 12abc"},
+ {"9223372036854775808", "1:1: integer 9223372036854775808 does not fit in 64 bits"},
+ {"1.0e400", "1:1: float 1.0e400 is out of range"},
+
+ // Lexer errors surface unchanged.
+ {`(a "oops`, "1:4: unterminated string"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.src, func(t *testing.T) {
+ _, err := ReadAll(tt.src)
+ var se *SyntaxError
+ if !errors.As(err, &se) {
+ t.Fatalf("ReadAll(%q) error = %v, want SyntaxError", tt.src, err)
+ }
+ if err.Error() != tt.want {
+ t.Errorf("ReadAll(%q) error = %q, want %q", tt.src, err.Error(), tt.want)
+ }
+ })
+ }
+}
+
+func TestReadDeepNesting(t *testing.T) {
+ const depth = 10_000
+ src := strings.Repeat("(", depth) + strings.Repeat(")", depth)
+ v := readOne(t, src)
+ for i := 0; i < depth-1; i++ {
+ v = v.(*Pair).Car
+ }
+ if v != Value(Nil) {
+ t.Errorf("innermost value = %s, want ()", Print(v))
+ }
+}
+
+func readOne(t *testing.T, src string) Value {
+ t.Helper()
+ forms, err := ReadAll(src)
+ if err != nil {
+ t.Fatalf("ReadAll(%q): %v", src, err)
+ }
+ if len(forms) != 1 {
+ t.Fatalf("ReadAll(%q) returned %d forms, want 1", src, len(forms))
+ }
+ return forms[0]
+}