government

commit 820eaee1ebb5edff55abac8b7a41579acbc49137

tree

parent:
307ad2c9a167e314a75ac46d5ee0382f494bf1cb

nmyk <nick@nmyk.io>

2026-11-14T11:23:05-05:00

add reader

diff --git a/sexpr/reader.go b/sexpr/reader.go
new file mode 100644
index 0000000000000000000000000000000000000000..c7edd7e2478425c515339af7003b53f7e461b02b
--- /dev/null
+++ b/sexpr/reader.go
@@ -0,0 +1,208 @@
+package sexpr
+
+import (
+	"fmt"
+	"math"
+	"regexp"
+	"strconv"
+)
+
+// Reader parses source text into values by recursive descent over the
+// grammar:
+//
+//	datum = atom | string | "'" datum | "(" list
+//	list  = ")" | datum { datum } [ "." datum ] ")"
+//
+// Comments are skipped for now; step 7.1 keeps them as notes.
+type Reader struct {
+	lex    *Lexer
+	peeked *Token
+}
+
+func NewReader(src string) *Reader {
+	return &Reader{lex: NewLexer(src)}
+}
+
+// ReadAll returns every top-level form in src.
+func ReadAll(src string) ([]Value, error) {
+	r := NewReader(src)
+	var forms []Value
+	for {
+		v, err := r.Read()
+		if err != nil {
+			return nil, err
+		}
+		if v == nil {
+			return forms, nil
+		}
+		forms = append(forms, v)
+	}
+}
+
+// Read returns the next top-level form, or nil at end of input.
+func (r *Reader) Read() (Value, error) {
+	t, err := r.peek()
+	if err != nil {
+		return nil, err
+	}
+	if t.Kind == TokEOF {
+		return nil, nil
+	}
+	return r.datum()
+}
+
+func (r *Reader) datum() (Value, error) {
+	t, err := r.next()
+	if err != nil {
+		return nil, err
+	}
+	switch t.Kind {
+	case TokAtom:
+		return parseAtom(t)
+	case TokString:
+		return String(t.Text), nil
+	case TokQuote:
+		next, err := r.peek()
+		if err != nil {
+			return nil, err
+		}
+		if next.Kind == TokEOF || next.Kind == TokRParen {
+			return nil, &SyntaxError{Pos: t.Pos, Msg: "' must be followed by a value"}
+		}
+		v, err := r.datum()
+		if err != nil {
+			return nil, err
+		}
+		return List(Symbol("quote"), v), nil
+	case TokLParen:
+		return r.list(t.Pos)
+	case TokRParen:
+		return nil, &SyntaxError{Pos: t.Pos, Msg: "unexpected )"}
+	case TokDot:
+		return nil, &SyntaxError{Pos: t.Pos, Msg: "unexpected . outside a list"}
+	}
+	panic(fmt.Sprintf("sexpr: reader got token %v", t.Kind))
+}
+
+// list parses the rest of a list after its "(". Errors about a missing ")"
+// point at the "(" that opened the list, which is where the mistake usually
+// is.
+func (r *Reader) list(open Pos) (Value, error) {
+	var items []Value
+	for {
+		t, err := r.peek()
+		if err != nil {
+			return nil, err
+		}
+		switch t.Kind {
+		case TokEOF:
+			return nil, &SyntaxError{Pos: open, Msg: "unterminated list"}
+		case TokRParen:
+			r.next()
+			return List(items...), nil
+		case TokDot:
+			if len(items) == 0 {
+				return nil, &SyntaxError{Pos: t.Pos, Msg: "a dotted pair needs a value before ."}
+			}
+			r.next()
+			return r.dottedTail(t.Pos, open, items)
+		}
+		v, err := r.datum()
+		if err != nil {
+			return nil, err
+		}
+		items = append(items, v)
+	}
+}
+
+// dottedTail parses the single value after "." and the ")" that must follow
+// it: a pair has exactly one cdr.
+func (r *Reader) dottedTail(dot, open Pos, items []Value) (Value, error) {
+	t, err := r.peek()
+	if err != nil {
+		return nil, err
+	}
+	switch t.Kind {
+	case TokRParen, TokEOF:
+		return nil, &SyntaxError{Pos: dot, Msg: "expected a value after ."}
+	case TokDot:
+		return nil, &SyntaxError{Pos: t.Pos, Msg: "only one . is allowed in a list"}
+	}
+	tail, err := r.datum()
+	if err != nil {
+		return nil, err
+	}
+	t, err = r.next()
+	if err != nil {
+		return nil, err
+	}
+	switch t.Kind {
+	case TokRParen:
+		return ListWithTail(tail, items...), nil
+	case TokEOF:
+		return nil, &SyntaxError{Pos: open, Msg: "unterminated list"}
+	case TokDot:
+		return nil, &SyntaxError{Pos: t.Pos, Msg: "only one . is allowed in a list"}
+	}
+	return nil, &SyntaxError{Pos: t.Pos, Msg: "expected ) after the value following ."}
+}
+
+// next and peek give the reader one token of lookahead, which is all this
+// grammar needs (it is LL(1)). Comments are dropped here.
+func (r *Reader) next() (Token, error) {
+	if r.peeked != nil {
+		t := *r.peeked
+		r.peeked = nil
+		return t, nil
+	}
+	for {
+		t, err := r.lex.Next()
+		if err != nil || t.Kind != TokComment {
+			return t, err
+		}
+	}
+}
+
+func (r *Reader) peek() (Token, error) {
+	if r.peeked == nil {
+		t, err := r.next()
+		if err != nil {
+			return Token{}, err
+		}
+		r.peeked = &t
+	}
+	return *r.peeked, nil
+}
+
+var (
+	intPattern   = regexp.MustCompile(`^-?(0|[1-9][0-9]*)$`)
+	floatPattern = regexp.MustCompile(`^-?[0-9]+\.[0-9]+([eE][-+]?[0-9]+)?$`)
+	// Anything starting like a number must be a valid one, so "007", "1."
+	// and ".5" are errors rather than surprising symbols.
+	numberLike = regexp.MustCompile(`^[-+]?\.?[0-9]`)
+)
+
+func parseAtom(t Token) (Value, error) {
+	s := t.Text
+	switch {
+	case s == "T":
+		return T, nil
+	case s == "F":
+		return F, nil
+	case intPattern.MatchString(s):
+		n, err := strconv.ParseInt(s, 10, 64)
+		if err != nil {
+			return nil, &SyntaxError{Pos: t.Pos, Msg: fmt.Sprintf("integer %s does not fit in 64 bits", s)}
+		}
+		return Int(n), nil
+	case floatPattern.MatchString(s):
+		f, err := strconv.ParseFloat(s, 64)
+		if err != nil || math.IsInf(f, 0) {
+			return nil, &SyntaxError{Pos: t.Pos, Msg: fmt.Sprintf("float %s is out of range", s)}
+		}
+		return Float(f), nil
+	case numberLike.MatchString(s):
+		return nil, &SyntaxError{Pos: t.Pos, Msg: fmt.Sprintf("malformed number %s", s)}
+	}
+	return Symbol(s), nil
+}
diff --git a/sexpr/reader_test.go b/sexpr/reader_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..9fe4b5634a56e2bb00089fa44ff3a5fe88c1d0e5
--- /dev/null
+++ b/sexpr/reader_test.go
@@ -0,0 +1,202 @@
+package sexpr
+
+import (
+	"errors"
+	"math"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestReadAtoms(t *testing.T) {
+	tests := []struct {
+		src  string
+		want Value
+	}{
+		{"0", Int(0)},
+		{"42", Int(42)},
+		{"-42", Int(-42)},
+		{"9223372036854775807", Int(math.MaxInt64)},
+		{"-9223372036854775808", Int(math.MinInt64)},
+		{"3.14", Float(3.14)},
+		{"-0.5", Float(-0.5)},
+		{"1.0", Float(1)},
+		{"1.5e3", Float(1500)},
+		{"1.0E-2", Float(0.01)},
+		{"1.0e+21", Float(1e21)},
+		{"T", T},
+		{"F", F},
+		{"t", Symbol("t")},
+		{"foo", Symbol("foo")},
+		{"+", Symbol("+")},
+		{"-", Symbol("-")},
+		{"<=", Symbol("<=")},
+		{"str?", Symbol("str?")},
+		{"-x", Symbol("-x")},
+		{"a.b", Symbol("a.b")},
+		{"x1", Symbol("x1")},
+		{"héllo", Symbol("héllo")},
+		{`"hi"`, String("hi")},
+		{`""`, String("")},
+		{`"1"`, String("1")},
+	}
+	for _, tt := range tests {
+		t.Run(tt.src, func(t *testing.T) {
+			got := readOne(t, tt.src)
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Read(%s) = %#v, want %#v", tt.src, got, tt.want)
+			}
+		})
+	}
+}
+
+// The printed form is the normal form: reading then printing gives it.
+func TestReadNormalizes(t *testing.T) {
+	tests := []struct{ src, want string }{
+		{"()", "()"},
+		{"( )", "()"},
+		{"(1 2 3)", "(1 2 3)"},
+		{"  ( 1\n\t2   3 )  ", "(1 2 3)"},
+		{"(a (b (c)) ())", "(a (b (c)) ())"},
+		{"(1 . 2)", "(1 . 2)"},
+		{"(1 2 . 3)", "(1 2 . 3)"},
+		{"(1 . (2 3))", "(1 2 3)"},
+		{"(1 . (2 . (3 . ())))", "(1 2 3)"},
+		{"(1 . ())", "(1)"},
+		{"((1 . 2) . 3)", "((1 . 2) . 3)"},
+		{"'x", "(quote x)"},
+		{"'(1 2)", "(quote (1 2))"},
+		{"''x", "(quote (quote x))"},
+		{"(a 'b c)", "(a (quote b) c)"},
+		{"'(1 . 2)", "(quote (1 . 2))"},
+		{"(1 . 'x)", "(1 quote x)"},
+		{"(+ 1 ; comment\n 2)", "(+ 1 2)"},
+		{"(define sq (lambda (x) (* x x)))", "(define sq (lambda (x) (* x x)))"},
+		{`("a\nb" "\u{41}")`, `("a\nb" "A")`},
+		{"1.50", "1.5"},
+		{"(T F)", "(T F)"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.src, func(t *testing.T) {
+			if got := Print(readOne(t, tt.src)); got != tt.want {
+				t.Errorf("Print(Read(%q)) = %s, want %s", tt.src, got, tt.want)
+			}
+		})
+	}
+}
+
+// Print's output must read back to an equal value.
+func TestPrintReadRoundTrip(t *testing.T) {
+	values := []Value{
+		Int(math.MinInt64), Int(math.MaxInt64), Float(0.1), Float(1), Float(-0.0),
+		Float(1e21), Float(1.5e-7), Float(math.MaxFloat64), Float(math.SmallestNonzeroFloat64),
+		String(""), String("a\"b\\c\nd\te"), String("bell\a\x00\x7f"), String("héllo 😀"),
+		T, F, Symbol("x"), Nil,
+		List(Int(1), List(Float(2), Nil), String("s")),
+		Cons(Int(1), Int(2)),
+		ListWithTail(Symbol("c"), Symbol("a"), Symbol("b")),
+		Cons(Cons(Int(1), Int(2)), Int(3)),
+	}
+	for _, v := range values {
+		printed := Print(v)
+		t.Run(printed, func(t *testing.T) {
+			if got := readOne(t, printed); !reflect.DeepEqual(got, v) {
+				t.Errorf("Read(Print(v)) = %s, want %s", Print(got), printed)
+			}
+		})
+	}
+}
+
+func TestReadAll(t *testing.T) {
+	forms, err := ReadAll("(define x 1) ; one\nx\n'y\n")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for _, f := range forms {
+		got = append(got, Print(f))
+	}
+	want := []string{"(define x 1)", "x", "(quote y)"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("ReadAll = %v, want %v", got, want)
+	}
+
+	forms, err = ReadAll("  ; only a comment\n")
+	if err != nil || len(forms) != 0 {
+		t.Errorf("ReadAll(comment only) = %v, %v; want no forms", forms, err)
+	}
+}
+
+func TestReadErrors(t *testing.T) {
+	tests := []struct{ src, want string }{
+		{")", "1:1: unexpected )"},
+		{"(1 2))", "1:6: unexpected )"},
+		{"(1 2", "1:1: unterminated list"},
+		{"(a\n  (b c)", "1:1: unterminated list"},
+		{"(a (b", "1:4: unterminated list"},
+		{".", "1:1: unexpected . outside a list"},
+		{"'", "1:1: ' must be followed by a value"},
+		{"(')", "1:2: ' must be followed by a value"},
+		{"'.", "1:2: unexpected . outside a list"},
+
+		// A pair has exactly one cdr.
+		{"(. 1)", "1:2: a dotted pair needs a value before ."},
+		{"(1 .)", "1:4: expected a value after ."},
+		{"(1 . )", "1:4: expected a value after ."},
+		{"(1 .", "1:4: expected a value after ."},
+		{"(1 . 2 3)", "1:8: expected ) after the value following ."},
+		{"(1 . 2 . 3)", "1:8: only one . is allowed in a list"},
+		{"(1 . . 2)", "1:6: only one . is allowed in a list"},
+		{"(1 . 2", "1:1: unterminated list"},
+
+		// Numbers must match the grammar exactly.
+		{"007", "1:1: malformed number 007"},
+		{"1.", "1:1: malformed number 1."},
+		{".5", "1:1: malformed number .5"},
+		{"-.5", "1:1: malformed number -.5"},
+		{"+5", "1:1: malformed number +5"},
+		{"1e5", "1:1: malformed number 1e5"},
+		{"12abc", "1:1: malformed number 12abc"},
+		{"9223372036854775808", "1:1: integer 9223372036854775808 does not fit in 64 bits"},
+		{"1.0e400", "1:1: float 1.0e400 is out of range"},
+
+		// Lexer errors surface unchanged.
+		{`(a "oops`, "1:4: unterminated string"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.src, func(t *testing.T) {
+			_, err := ReadAll(tt.src)
+			var se *SyntaxError
+			if !errors.As(err, &se) {
+				t.Fatalf("ReadAll(%q) error = %v, want SyntaxError", tt.src, err)
+			}
+			if err.Error() != tt.want {
+				t.Errorf("ReadAll(%q) error = %q, want %q", tt.src, err.Error(), tt.want)
+			}
+		})
+	}
+}
+
+func TestReadDeepNesting(t *testing.T) {
+	const depth = 10_000
+	src := strings.Repeat("(", depth) + strings.Repeat(")", depth)
+	v := readOne(t, src)
+	for i := 0; i < depth-1; i++ {
+		v = v.(*Pair).Car
+	}
+	if v != Value(Nil) {
+		t.Errorf("innermost value = %s, want ()", Print(v))
+	}
+}
+
+func readOne(t *testing.T, src string) Value {
+	t.Helper()
+	forms, err := ReadAll(src)
+	if err != nil {
+		t.Fatalf("ReadAll(%q): %v", src, err)
+	}
+	if len(forms) != 1 {
+		t.Fatalf("ReadAll(%q) returned %d forms, want 1", src, len(forms))
+	}
+	return forms[0]
+}