government

commit 405825f89f1aafd129e6aa5b5bd6efeebe66616e

tree

parent:
cdf2d0cb24bc5f0af81926cf5d0b2a709fd84abc

nmyk <nick@nmyk.io>

2026-11-14T15:40:21-05:00

add tree unlayering

diff --git a/ast/layer.go b/ast/layer.go
index e0f5589f548468e1eed43dcc84835c8a7dbde691..457a3cdf6f8fcadea23452c9352bb72e5f023139 100644
--- a/ast/layer.go
+++ b/ast/layer.go
@@ -77,7 +77,7 @@
 	// Certificates list policies and mappings in id order, so the same
 	// program always produces the same bytes.
 	for d := range l.Nodes {
-		slices.SortFunc(l.Nodes[d], func(a, b *Node) int { return cmp.Compare(a.ID, b.ID) })
+		sortByID(l.Nodes[d])
 		slices.SortFunc(l.Edges[d], func(a, b Edge) int {
 			return cmp.Or(cmp.Compare(a.Parent.ID, b.Parent.ID), cmp.Compare(a.Child.ID, b.Child.ID))
 		})
diff --git a/ast/tree.go b/ast/tree.go
index 39464736312426ebd1c4e19776400f13d1cd7e64..61bfe7459a7f9777d6fa8e8bf78458eb82b19c55 100644
--- a/ast/tree.go
+++ b/ast/tree.go
@@ -4,6 +4,7 @@ import (
 	"crypto/x509"
 	"fmt"
 	"strconv"
+	"strings"
 
 	"government/sexpr"
 )
@@ -152,11 +153,11 @@ 	return forms, origins, nil
 }
 
 func value(n *Node, origins Origins) (sexpr.Value, error) {
+	if err := checkShape(n); err != nil {
+		return nil, err
+	}
 	if n.IsList() {
 		return listValue(n, origins)
-	}
-	if len(n.Children) > 0 {
-		return nil, fmt.Errorf("node %d: %s %q cannot have children", n.ID, n.NS, n.Text)
 	}
 	switch n.NS {
 	case NSBuiltin:
@@ -168,7 +169,7 @@ 			return sexpr.F, nil
 		case Empty:
 			return sexpr.Nil, nil
 		case Dot:
-			return nil, fmt.Errorf("node %d: . is only allowed second-to-last in a list", n.ID)
+			return nil, fmt.Errorf("%s is outside a list; . belongs second-to-last in one", n)
 		}
 		return sexpr.Symbol(n.Text), nil
 	case NSSymbol:
@@ -176,19 +177,16 @@ 		return sexpr.Symbol(n.Text), nil
 	case NSInt, NSFloat:
 		v, _, err := sexpr.ParseNumber(n.Text)
 		if err != nil {
-			return nil, fmt.Errorf("node %d: %w", n.ID, err)
+			return nil, fmt.Errorf("%s: %w", n, err)
 		}
 		return v, nil
 	case NSString:
 		return sexpr.String(n.Text), nil
 	}
-	return nil, fmt.Errorf("node %d: unknown %s", n.ID, n.NS)
+	return nil, fmt.Errorf("%s: unknown %s", n, n.NS)
 }
 
 func listValue(n *Node, origins Origins) (sexpr.Value, error) {
-	if len(n.Children) == 0 {
-		return nil, fmt.Errorf("node %d: a list node needs at least one element; () is its own builtin", n.ID)
-	}
 	children := n.Children
 	var tail sexpr.Value = sexpr.Nil
 	if k := len(children); k >= 3 && isDot(children[k-2]) {
@@ -214,3 +212,35 @@
 func isDot(n *Node) bool {
 	return n.NS == NSBuiltin && n.Text == Dot
 }
+
+// checkShape enforces the decoding rules that concern a node and its own
+// children: a list has at least one element, an atom has none, and "."
+// sits second-to-last with at least one element before it.
+func checkShape(n *Node) error {
+	if !n.IsList() {
+		if len(n.Children) > 0 {
+			return fmt.Errorf("atom %s has child %s; an atom can have none", n, n.Children[0])
+		}
+		return nil
+	}
+	k := len(n.Children)
+	if k == 0 {
+		return fmt.Errorf("list %s has no elements; the empty list is the () builtin", n)
+	}
+	for i, c := range n.Children {
+		if isDot(c) && (i != k-2 || k < 3) {
+			return fmt.Errorf("list %s has . at position %d of %d; it belongs second-to-last, after at least one element", n, i+1, k)
+		}
+	}
+	return nil
+}
+
+// String names a node the way tools print OIDs, with GOV for Root, so an
+// error points at something visible in a certificate dump.
+func (n *Node) String() string {
+	oid, err := n.OID()
+	if err != nil {
+		return fmt.Sprintf("#%d (%s %q)", n.ID, n.NS, n.Text)
+	}
+	return strings.Replace(oid.String(), Root, "GOV", 1)
+}
diff --git a/ast/tree_test.go b/ast/tree_test.go
index 05c43b83c2dc499998945ebee1fb060d9f2a866d..b4d30a2fb62d3a2ed4d2a33faa333611add77d52 100644
--- a/ast/tree_test.go
+++ b/ast/tree_test.go
@@ -198,11 +198,11 @@ 		name string
 		form *Node
 		want string
 	}{
-		{"empty list node", list(1), "at least one element"},
-		{"atom with a child", &Node{ID: 1, NS: NSInt, Text: "1", Children: []*Node{atom(2, NSInt, "2")}}, "cannot have children"},
-		{"dot at the start", list(1, atom(2, NSBuiltin, Dot), atom(3, NSSymbol, "a")), "second-to-last"},
-		{"dot at the end", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot)), "second-to-last"},
-		{"two tails", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot), atom(4, NSSymbol, "b"), atom(5, NSSymbol, "c")), "second-to-last"},
+		{"empty list node", list(1), "no elements"},
+		{"atom with a child", &Node{ID: 1, NS: NSInt, Text: "1", Children: []*Node{atom(2, NSInt, "2")}}, "an atom can have none"},
+		{"dot at the start", list(1, atom(2, NSBuiltin, Dot), atom(3, NSSymbol, "a")), "position 1 of 2"},
+		{"dot at the end", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot)), "position 2 of 2"},
+		{"two tails", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot), atom(4, NSSymbol, "b"), atom(5, NSSymbol, "c")), "position 2 of 4"},
 		{"bad integer", atom(1, NSInt, "007"), "malformed"},
 	}
 	for _, tt := range tests {
diff --git a/ast/unlayer.go b/ast/unlayer.go
new file mode 100644
index 0000000000000000000000000000000000000000..6b9d9fc4543f0a54931a07716996d74e51693eda
--- /dev/null
+++ b/ast/unlayer.go
@@ -0,0 +1,144 @@
+package ast
+
+import (
+	"cmp"
+	"fmt"
+	"slices"
+)
+
+// Unlayer rebuilds a tree from its layers. It is how a validated policy
+// graph becomes a program, so its input is untrusted: it reads only each
+// node's ID, NS and Text, takes structure from the edges alone, and checks
+// every decoding rule in DESIGN.md before returning.
+//
+// A node may have several parents. The result then shares that node rather
+// than duplicating it, which is how shared subexpressions survive decoding.
+func Unlayer(l Layers) (*Tree, error) {
+	u := &unlayerer{
+		depth:    map[*Node]int{},
+		children: map[*Node][]*Node{},
+		built:    map[*Node]*Node{},
+		byID:     map[uint64]*Node{},
+	}
+	if err := u.index(l); err != nil {
+		return nil, err
+	}
+	if len(l.Nodes) == 0 || len(l.Nodes[0]) == 0 {
+		return nil, fmt.Errorf("no top-level forms")
+	}
+
+	forms := slices.Clone(l.Nodes[0])
+	sortByID(forms)
+	t := &Tree{}
+	for _, f := range forms {
+		if isDot(f) {
+			return nil, fmt.Errorf("depth 1: %s is outside a list; . belongs second-to-last in one", f)
+		}
+		n, err := u.build(f)
+		if err != nil {
+			return nil, err
+		}
+		t.Forms = append(t.Forms, n)
+	}
+	return t, nil
+}
+
+type unlayerer struct {
+	depth    map[*Node]int
+	children map[*Node][]*Node
+	built    map[*Node]*Node // input node -> output node, so shared nodes stay shared
+	byID     map[uint64]*Node
+}
+
+// index records each node's depth and children, and checks that every
+// edge goes down exactly one level and every node below depth 1 has a
+// parent.
+func (u *unlayerer) index(l Layers) error {
+	for d, nodes := range l.Nodes {
+		for _, n := range nodes {
+			if prev, ok := u.depth[n]; ok {
+				return fmt.Errorf("%s is listed at depths %d and %d", n, prev, d+1)
+			}
+			u.depth[n] = d + 1
+		}
+	}
+	hasParent := map[*Node]bool{}
+	for d, edges := range l.Edges {
+		for _, e := range edges {
+			if u.depth[e.Parent] != d+1 || u.depth[e.Child] != d+2 {
+				return fmt.Errorf("cert %d: mapping %s -> %s does not go from depth %d to %d", d+1, e.Parent, e.Child, d+1, d+2)
+			}
+			u.children[e.Parent] = append(u.children[e.Parent], e.Child)
+			hasParent[e.Child] = true
+		}
+	}
+	for d := 1; d < len(l.Nodes); d++ {
+		for _, n := range l.Nodes[d] {
+			if !hasParent[n] {
+				return fmt.Errorf("depth %d: %s has no parent", d+1, n)
+			}
+		}
+	}
+	return nil
+}
+
+func (u *unlayerer) build(in *Node) (*Node, error) {
+	if out, ok := u.built[in]; ok {
+		return out, nil
+	}
+	d := u.depth[in]
+	out := &Node{ID: in.ID, NS: in.NS, Text: in.Text}
+	// Ids are unique across the program, so two real nodes with one id
+	// means two different OIDs claim the same place in argument order.
+	if other, ok := u.byID[in.ID]; ok {
+		return nil, fmt.Errorf("depth %d: %s and %s share node id %d", d, other, out, in.ID)
+	}
+	u.byID[in.ID] = out
+	u.built[in] = out
+
+	for _, c := range u.children[in] {
+		if sameOID(c, in) {
+			if err := u.checkCopy(c); err != nil {
+				return nil, err
+			}
+			continue
+		}
+		child, err := u.build(c)
+		if err != nil {
+			return nil, err
+		}
+		out.Children = append(out.Children, child)
+	}
+	sortByID(out.Children)
+	if err := checkShape(out); err != nil {
+		return nil, fmt.Errorf("depth %d: %w", d, err)
+	}
+	return out, nil
+}
+
+// checkCopy accepts a copy the validator made and everything below it,
+// which may only be further copies of the same atom. A real node under a
+// copy would mean an edge from an OID the program never mapped from at
+// that depth.
+func (u *unlayerer) checkCopy(c *Node) error {
+	for _, below := range u.children[c] {
+		if !sameOID(below, c) {
+			return fmt.Errorf("depth %d: copy %s has real child %s; a copy may only have copies below it", u.depth[c], c, below)
+		}
+		if err := u.checkCopy(below); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// sameOID is how the decoder recognizes the validator's copies: an atom
+// with no mapping expects only its own OID, so anyPolicy in the next cert
+// gives it a child with that same OID.
+func sameOID(a, b *Node) bool {
+	return a.ID == b.ID && a.NS == b.NS && a.Text == b.Text
+}
+
+func sortByID(nodes []*Node) {
+	slices.SortFunc(nodes, func(a, b *Node) int { return cmp.Compare(a.ID, b.ID) })
+}
diff --git a/ast/unlayer_test.go b/ast/unlayer_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..282069502f252b92b545e4d82d1caf12ed850b40
--- /dev/null
+++ b/ast/unlayer_test.go
@@ -0,0 +1,230 @@
+package ast
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+// withCopies adds what policy processing adds on top of Layer's output: a
+// copy of every node that has no children, one level down at each depth
+// until the bottom. This is the shape Unlayer sees in the real pipeline.
+func withCopies(l Layers) Layers {
+	n := l.Depth()
+	out := Layers{Nodes: make([][]*Node, n), Edges: make([][]Edge, n)}
+	for d := range l.Nodes {
+		out.Nodes[d] = append(out.Nodes[d], l.Nodes[d]...)
+		out.Edges[d] = append(out.Edges[d], l.Edges[d]...)
+	}
+	for d := 0; d < n-1; d++ {
+		for _, node := range out.Nodes[d] {
+			if len(childrenIn(out.Edges[d], node)) > 0 {
+				continue
+			}
+			c := &Node{ID: node.ID, NS: node.NS, Text: node.Text}
+			out.Nodes[d+1] = append(out.Nodes[d+1], c)
+			out.Edges[d] = append(out.Edges[d], Edge{node, c})
+		}
+	}
+	return out
+}
+
+func childrenIn(edges []Edge, parent *Node) []*Node {
+	var kids []*Node
+	for _, e := range edges {
+		if e.Parent == parent {
+			kids = append(kids, e.Child)
+		}
+	}
+	return kids
+}
+
+func roundTrip(t *testing.T, name, src string) {
+	t.Helper()
+	tree, err := ToTree(mustRead(t, src))
+	if err != nil {
+		t.Fatalf("%s: ToTree: %v", name, err)
+	}
+	l, err := Layer(tree)
+	if err != nil {
+		t.Fatalf("%s: Layer: %v", name, err)
+	}
+	for _, input := range []struct {
+		label  string
+		layers Layers
+	}{{"layers", l}, {"layers with copies", withCopies(l)}} {
+		got, err := Unlayer(input.layers)
+		if err != nil {
+			t.Errorf("%s: Unlayer(%s): %v", name, input.label, err)
+			continue
+		}
+		if !reflect.DeepEqual(got, tree) {
+			t.Errorf("%s: Unlayer(%s) differs from the tree", name, input.label)
+		}
+	}
+}
+
+func TestUnlayerRoundTripTestdata(t *testing.T) {
+	files, err := filepath.Glob("../testdata/*.lisp")
+	if err != nil || len(files) == 0 {
+		t.Fatalf("no testdata: %v", err)
+	}
+	for _, file := range files {
+		src, err := os.ReadFile(file)
+		if err != nil {
+			t.Fatal(err)
+		}
+		roundTrip(t, file, string(src))
+	}
+}
+
+func TestUnlayerRoundTripShapes(t *testing.T) {
+	for _, src := range []string{
+		"(+ 1 (* 2 3))",
+		"1 (a (b (c)))",
+		"'(1 2 . 3)",
+		"'((a . b) . c)",
+		"(f () T F)",
+	} {
+		roundTrip(t, src, src)
+	}
+}
+
+func TestUnlayerCopiesAreSkipped(t *testing.T) {
+	l := withCopies(mustLayer(t, "(+ 1 (* 2 3))"))
+	// The validator's graph has the design's two copies, +#2' and 1#3'.
+	if got := len(l.Nodes[2]); got != 5 {
+		t.Fatalf("depth 3 has %d nodes, want 5", got)
+	}
+	tree, err := Unlayer(l)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got := outline(tree.Forms); got != "(#1 +#2 1#3 (#4 *#5 2#6 3#7))" {
+		t.Errorf("Unlayer = %s", got)
+	}
+}
+
+func TestUnlayerKeepsSharedNodes(t *testing.T) {
+	x := &Node{ID: 3, NS: NSSymbol, Text: "x"}
+	f := &Node{ID: 2, NS: NSSymbol, Text: "f"}
+	g := &Node{ID: 5, NS: NSSymbol, Text: "g"}
+	a := &Node{ID: 1, NS: NSBuiltin, Text: ListNode}
+	b := &Node{ID: 4, NS: NSBuiltin, Text: ListNode}
+	l := Layers{
+		Nodes: [][]*Node{{a, b}, {f, x, g}},
+		Edges: [][]Edge{{{a, f}, {a, x}, {b, g}, {b, x}}, nil},
+	}
+	tree, err := Unlayer(l)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if tree.Forms[0].Children[1] != tree.Forms[1].Children[0] {
+		t.Error("x was duplicated instead of shared")
+	}
+}
+
+func TestUnlayerRejects(t *testing.T) {
+	node := func(id uint64, ns Namespace, text string) *Node { return &Node{ID: id, NS: ns, Text: text} }
+	list := func(id uint64) *Node { return node(id, NSBuiltin, ListNode) }
+	copyOf := func(n *Node) *Node { return node(n.ID, n.NS, n.Text) }
+
+	one := node(2, NSInt, "1")
+	oneCopy := copyOf(one)
+	stray := node(3, NSInt, "2")
+	root := list(1)
+
+	tests := []struct {
+		name   string
+		layers Layers
+		want   string
+	}{
+		{
+			"copy with a real child",
+			Layers{
+				Nodes: [][]*Node{{root}, {one}, {oneCopy}, {stray}},
+				Edges: [][]Edge{{{root, one}}, {{one, oneCopy}}, {{oneCopy, stray}}, nil},
+			},
+			"copy GOV.2.2.49 has real child GOV.2.3.50",
+		},
+		{
+			"atom with a real child",
+			Layers{
+				Nodes: [][]*Node{{root}, {one}, {stray}},
+				Edges: [][]Edge{{{root, one}}, {{one, stray}}, nil},
+			},
+			"atom GOV.2.2.49 has child GOV.2.3.50",
+		},
+		{
+			"list node with no elements",
+			Layers{Nodes: [][]*Node{{root}}, Edges: [][]Edge{nil}},
+			"list GOV.0.1.1 has no elements",
+		},
+		{
+			"list node whose only child is a copy",
+			func() Layers {
+				r, c := list(1), list(1)
+				return Layers{Nodes: [][]*Node{{r}, {c}}, Edges: [][]Edge{{{r, c}}, nil}}
+			}(),
+			"list GOV.0.1.1 has no elements",
+		},
+		{
+			". first in a list",
+			func() Layers {
+				r, dot, a := list(1), node(2, NSBuiltin, Dot), node(3, NSSymbol, "a")
+				return Layers{Nodes: [][]*Node{{r}, {dot, a}}, Edges: [][]Edge{{{r, dot}, {r, a}}, nil}}
+			}(),
+			". at position 1 of 2",
+		},
+		{
+			". as a top-level form",
+			Layers{Nodes: [][]*Node{{node(1, NSBuiltin, Dot)}}, Edges: [][]Edge{nil}},
+			"depth 1: GOV.0.1.2 is outside a list",
+		},
+		{
+			"node with no parent",
+			Layers{
+				Nodes: [][]*Node{{root}, {one, stray}},
+				Edges: [][]Edge{{{root, one}}, nil},
+			},
+			"depth 2: GOV.2.3.50 has no parent",
+		},
+		{
+			"mapping that skips a depth",
+			Layers{
+				Nodes: [][]*Node{{root}, {one}, {stray}},
+				Edges: [][]Edge{{{root, one}, {root, stray}}, nil, nil},
+			},
+			"cert 1: mapping GOV.0.1.1 -> GOV.2.3.50 does not go from depth 1 to 2",
+		},
+		{
+			"two nodes with one id",
+			func() Layers {
+				r, a, b := list(1), node(2, NSSymbol, "a"), node(2, NSSymbol, "b")
+				return Layers{Nodes: [][]*Node{{r}, {a, b}}, Edges: [][]Edge{{{r, a}, {r, b}}, nil}}
+			}(),
+			"share node id 2",
+		},
+		{
+			"no forms",
+			Layers{},
+			"no top-level forms",
+		},
+	}
+	seen := map[string]string{}
+	for _, tt := range tests {
+		_, err := Unlayer(tt.layers)
+		if err == nil || !strings.Contains(err.Error(), tt.want) {
+			t.Errorf("%s: error = %v, want %q", tt.name, err, tt.want)
+			continue
+		}
+		// Cases that expect the same message are the same mistake; any other
+		// pair must be told apart.
+		if prev, dup := seen[err.Error()]; dup && prev != tt.want {
+			t.Errorf("%s gives the same error as a different case: %v", tt.name, err)
+		}
+		seen[err.Error()] = tt.want
+	}
+}