parent:
cdf2d0cb24bc5f0af81926cf5d0b2a709fd84abc
nmyk <nick@nmyk.io>
2026-11-14T15:40:21-05:00
add tree unlayering
diff --git a/ast/layer.go b/ast/layer.go
index e0f5589f548468e1eed43dcc84835c8a7dbde691..457a3cdf6f8fcadea23452c9352bb72e5f023139 100644
--- a/ast/layer.go
+++ b/ast/layer.go
@@ -77,7 +77,7 @@
// Certificates list policies and mappings in id order, so the same
// program always produces the same bytes.
for d := range l.Nodes {
- slices.SortFunc(l.Nodes[d], func(a, b *Node) int { return cmp.Compare(a.ID, b.ID) })
+ sortByID(l.Nodes[d])
slices.SortFunc(l.Edges[d], func(a, b Edge) int {
return cmp.Or(cmp.Compare(a.Parent.ID, b.Parent.ID), cmp.Compare(a.Child.ID, b.Child.ID))
})
diff --git a/ast/tree.go b/ast/tree.go
index 39464736312426ebd1c4e19776400f13d1cd7e64..61bfe7459a7f9777d6fa8e8bf78458eb82b19c55 100644
--- a/ast/tree.go
+++ b/ast/tree.go
@@ -4,6 +4,7 @@ import (
"crypto/x509"
"fmt"
"strconv"
+ "strings"
"government/sexpr"
)
@@ -152,11 +153,11 @@ return forms, origins, nil
}
func value(n *Node, origins Origins) (sexpr.Value, error) {
+ if err := checkShape(n); err != nil {
+ return nil, err
+ }
if n.IsList() {
return listValue(n, origins)
- }
- if len(n.Children) > 0 {
- return nil, fmt.Errorf("node %d: %s %q cannot have children", n.ID, n.NS, n.Text)
}
switch n.NS {
case NSBuiltin:
@@ -168,7 +169,7 @@ return sexpr.F, nil
case Empty:
return sexpr.Nil, nil
case Dot:
- return nil, fmt.Errorf("node %d: . is only allowed second-to-last in a list", n.ID)
+ return nil, fmt.Errorf("%s is outside a list; . belongs second-to-last in one", n)
}
return sexpr.Symbol(n.Text), nil
case NSSymbol:
@@ -176,19 +177,16 @@ return sexpr.Symbol(n.Text), nil
case NSInt, NSFloat:
v, _, err := sexpr.ParseNumber(n.Text)
if err != nil {
- return nil, fmt.Errorf("node %d: %w", n.ID, err)
+ return nil, fmt.Errorf("%s: %w", n, err)
}
return v, nil
case NSString:
return sexpr.String(n.Text), nil
}
- return nil, fmt.Errorf("node %d: unknown %s", n.ID, n.NS)
+ return nil, fmt.Errorf("%s: unknown %s", n, n.NS)
}
func listValue(n *Node, origins Origins) (sexpr.Value, error) {
- if len(n.Children) == 0 {
- return nil, fmt.Errorf("node %d: a list node needs at least one element; () is its own builtin", n.ID)
- }
children := n.Children
var tail sexpr.Value = sexpr.Nil
if k := len(children); k >= 3 && isDot(children[k-2]) {
@@ -214,3 +212,35 @@
func isDot(n *Node) bool {
return n.NS == NSBuiltin && n.Text == Dot
}
+
+// checkShape enforces the decoding rules that concern a node and its own
+// children: a list has at least one element, an atom has none, and "."
+// sits second-to-last with at least one element before it.
+func checkShape(n *Node) error {
+ if !n.IsList() {
+ if len(n.Children) > 0 {
+ return fmt.Errorf("atom %s has child %s; an atom can have none", n, n.Children[0])
+ }
+ return nil
+ }
+ k := len(n.Children)
+ if k == 0 {
+ return fmt.Errorf("list %s has no elements; the empty list is the () builtin", n)
+ }
+ for i, c := range n.Children {
+ if isDot(c) && (i != k-2 || k < 3) {
+ return fmt.Errorf("list %s has . at position %d of %d; it belongs second-to-last, after at least one element", n, i+1, k)
+ }
+ }
+ return nil
+}
+
+// String names a node the way tools print OIDs, with GOV for Root, so an
+// error points at something visible in a certificate dump.
+func (n *Node) String() string {
+ oid, err := n.OID()
+ if err != nil {
+ return fmt.Sprintf("#%d (%s %q)", n.ID, n.NS, n.Text)
+ }
+ return strings.Replace(oid.String(), Root, "GOV", 1)
+}
diff --git a/ast/tree_test.go b/ast/tree_test.go
index 05c43b83c2dc499998945ebee1fb060d9f2a866d..b4d30a2fb62d3a2ed4d2a33faa333611add77d52 100644
--- a/ast/tree_test.go
+++ b/ast/tree_test.go
@@ -198,11 +198,11 @@ name string
form *Node
want string
}{
- {"empty list node", list(1), "at least one element"},
- {"atom with a child", &Node{ID: 1, NS: NSInt, Text: "1", Children: []*Node{atom(2, NSInt, "2")}}, "cannot have children"},
- {"dot at the start", list(1, atom(2, NSBuiltin, Dot), atom(3, NSSymbol, "a")), "second-to-last"},
- {"dot at the end", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot)), "second-to-last"},
- {"two tails", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot), atom(4, NSSymbol, "b"), atom(5, NSSymbol, "c")), "second-to-last"},
+ {"empty list node", list(1), "no elements"},
+ {"atom with a child", &Node{ID: 1, NS: NSInt, Text: "1", Children: []*Node{atom(2, NSInt, "2")}}, "an atom can have none"},
+ {"dot at the start", list(1, atom(2, NSBuiltin, Dot), atom(3, NSSymbol, "a")), "position 1 of 2"},
+ {"dot at the end", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot)), "position 2 of 2"},
+ {"two tails", list(1, atom(2, NSSymbol, "a"), atom(3, NSBuiltin, Dot), atom(4, NSSymbol, "b"), atom(5, NSSymbol, "c")), "position 2 of 4"},
{"bad integer", atom(1, NSInt, "007"), "malformed"},
}
for _, tt := range tests {
diff --git a/ast/unlayer.go b/ast/unlayer.go
new file mode 100644
index 0000000000000000000000000000000000000000..6b9d9fc4543f0a54931a07716996d74e51693eda
--- /dev/null
+++ b/ast/unlayer.go
@@ -0,0 +1,144 @@
+package ast
+
+import (
+ "cmp"
+ "fmt"
+ "slices"
+)
+
+// Unlayer rebuilds a tree from its layers. It is how a validated policy
+// graph becomes a program, so its input is untrusted: it reads only each
+// node's ID, NS and Text, takes structure from the edges alone, and checks
+// every decoding rule in DESIGN.md before returning.
+//
+// A node may have several parents. The result then shares that node rather
+// than duplicating it, which is how shared subexpressions survive decoding.
+func Unlayer(l Layers) (*Tree, error) {
+ u := &unlayerer{
+ depth: map[*Node]int{},
+ children: map[*Node][]*Node{},
+ built: map[*Node]*Node{},
+ byID: map[uint64]*Node{},
+ }
+ if err := u.index(l); err != nil {
+ return nil, err
+ }
+ if len(l.Nodes) == 0 || len(l.Nodes[0]) == 0 {
+ return nil, fmt.Errorf("no top-level forms")
+ }
+
+ forms := slices.Clone(l.Nodes[0])
+ sortByID(forms)
+ t := &Tree{}
+ for _, f := range forms {
+ if isDot(f) {
+ return nil, fmt.Errorf("depth 1: %s is outside a list; . belongs second-to-last in one", f)
+ }
+ n, err := u.build(f)
+ if err != nil {
+ return nil, err
+ }
+ t.Forms = append(t.Forms, n)
+ }
+ return t, nil
+}
+
+type unlayerer struct {
+ depth map[*Node]int
+ children map[*Node][]*Node
+ built map[*Node]*Node // input node -> output node, so shared nodes stay shared
+ byID map[uint64]*Node
+}
+
+// index records each node's depth and children, and checks that every
+// edge goes down exactly one level and every node below depth 1 has a
+// parent.
+func (u *unlayerer) index(l Layers) error {
+ for d, nodes := range l.Nodes {
+ for _, n := range nodes {
+ if prev, ok := u.depth[n]; ok {
+ return fmt.Errorf("%s is listed at depths %d and %d", n, prev, d+1)
+ }
+ u.depth[n] = d + 1
+ }
+ }
+ hasParent := map[*Node]bool{}
+ for d, edges := range l.Edges {
+ for _, e := range edges {
+ if u.depth[e.Parent] != d+1 || u.depth[e.Child] != d+2 {
+ return fmt.Errorf("cert %d: mapping %s -> %s does not go from depth %d to %d", d+1, e.Parent, e.Child, d+1, d+2)
+ }
+ u.children[e.Parent] = append(u.children[e.Parent], e.Child)
+ hasParent[e.Child] = true
+ }
+ }
+ for d := 1; d < len(l.Nodes); d++ {
+ for _, n := range l.Nodes[d] {
+ if !hasParent[n] {
+ return fmt.Errorf("depth %d: %s has no parent", d+1, n)
+ }
+ }
+ }
+ return nil
+}
+
+func (u *unlayerer) build(in *Node) (*Node, error) {
+ if out, ok := u.built[in]; ok {
+ return out, nil
+ }
+ d := u.depth[in]
+ out := &Node{ID: in.ID, NS: in.NS, Text: in.Text}
+ // Ids are unique across the program, so two real nodes with one id
+ // means two different OIDs claim the same place in argument order.
+ if other, ok := u.byID[in.ID]; ok {
+ return nil, fmt.Errorf("depth %d: %s and %s share node id %d", d, other, out, in.ID)
+ }
+ u.byID[in.ID] = out
+ u.built[in] = out
+
+ for _, c := range u.children[in] {
+ if sameOID(c, in) {
+ if err := u.checkCopy(c); err != nil {
+ return nil, err
+ }
+ continue
+ }
+ child, err := u.build(c)
+ if err != nil {
+ return nil, err
+ }
+ out.Children = append(out.Children, child)
+ }
+ sortByID(out.Children)
+ if err := checkShape(out); err != nil {
+ return nil, fmt.Errorf("depth %d: %w", d, err)
+ }
+ return out, nil
+}
+
+// checkCopy accepts a copy the validator made and everything below it,
+// which may only be further copies of the same atom. A real node under a
+// copy would mean an edge from an OID the program never mapped from at
+// that depth.
+func (u *unlayerer) checkCopy(c *Node) error {
+ for _, below := range u.children[c] {
+ if !sameOID(below, c) {
+ return fmt.Errorf("depth %d: copy %s has real child %s; a copy may only have copies below it", u.depth[c], c, below)
+ }
+ if err := u.checkCopy(below); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// sameOID is how the decoder recognizes the validator's copies: an atom
+// with no mapping expects only its own OID, so anyPolicy in the next cert
+// gives it a child with that same OID.
+func sameOID(a, b *Node) bool {
+ return a.ID == b.ID && a.NS == b.NS && a.Text == b.Text
+}
+
+func sortByID(nodes []*Node) {
+ slices.SortFunc(nodes, func(a, b *Node) int { return cmp.Compare(a.ID, b.ID) })
+}
diff --git a/ast/unlayer_test.go b/ast/unlayer_test.go
new file mode 100644
index 0000000000000000000000000000000000000000..282069502f252b92b545e4d82d1caf12ed850b40
--- /dev/null
+++ b/ast/unlayer_test.go
@@ -0,0 +1,230 @@
+package ast
+
+import (
+ "os"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+// withCopies adds what policy processing adds on top of Layer's output: a
+// copy of every node that has no children, one level down at each depth
+// until the bottom. This is the shape Unlayer sees in the real pipeline.
+func withCopies(l Layers) Layers {
+ n := l.Depth()
+ out := Layers{Nodes: make([][]*Node, n), Edges: make([][]Edge, n)}
+ for d := range l.Nodes {
+ out.Nodes[d] = append(out.Nodes[d], l.Nodes[d]...)
+ out.Edges[d] = append(out.Edges[d], l.Edges[d]...)
+ }
+ for d := 0; d < n-1; d++ {
+ for _, node := range out.Nodes[d] {
+ if len(childrenIn(out.Edges[d], node)) > 0 {
+ continue
+ }
+ c := &Node{ID: node.ID, NS: node.NS, Text: node.Text}
+ out.Nodes[d+1] = append(out.Nodes[d+1], c)
+ out.Edges[d] = append(out.Edges[d], Edge{node, c})
+ }
+ }
+ return out
+}
+
+func childrenIn(edges []Edge, parent *Node) []*Node {
+ var kids []*Node
+ for _, e := range edges {
+ if e.Parent == parent {
+ kids = append(kids, e.Child)
+ }
+ }
+ return kids
+}
+
+func roundTrip(t *testing.T, name, src string) {
+ t.Helper()
+ tree, err := ToTree(mustRead(t, src))
+ if err != nil {
+ t.Fatalf("%s: ToTree: %v", name, err)
+ }
+ l, err := Layer(tree)
+ if err != nil {
+ t.Fatalf("%s: Layer: %v", name, err)
+ }
+ for _, input := range []struct {
+ label string
+ layers Layers
+ }{{"layers", l}, {"layers with copies", withCopies(l)}} {
+ got, err := Unlayer(input.layers)
+ if err != nil {
+ t.Errorf("%s: Unlayer(%s): %v", name, input.label, err)
+ continue
+ }
+ if !reflect.DeepEqual(got, tree) {
+ t.Errorf("%s: Unlayer(%s) differs from the tree", name, input.label)
+ }
+ }
+}
+
+func TestUnlayerRoundTripTestdata(t *testing.T) {
+ files, err := filepath.Glob("../testdata/*.lisp")
+ if err != nil || len(files) == 0 {
+ t.Fatalf("no testdata: %v", err)
+ }
+ for _, file := range files {
+ src, err := os.ReadFile(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ roundTrip(t, file, string(src))
+ }
+}
+
+func TestUnlayerRoundTripShapes(t *testing.T) {
+ for _, src := range []string{
+ "(+ 1 (* 2 3))",
+ "1 (a (b (c)))",
+ "'(1 2 . 3)",
+ "'((a . b) . c)",
+ "(f () T F)",
+ } {
+ roundTrip(t, src, src)
+ }
+}
+
+func TestUnlayerCopiesAreSkipped(t *testing.T) {
+ l := withCopies(mustLayer(t, "(+ 1 (* 2 3))"))
+ // The validator's graph has the design's two copies, +#2' and 1#3'.
+ if got := len(l.Nodes[2]); got != 5 {
+ t.Fatalf("depth 3 has %d nodes, want 5", got)
+ }
+ tree, err := Unlayer(l)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := outline(tree.Forms); got != "(#1 +#2 1#3 (#4 *#5 2#6 3#7))" {
+ t.Errorf("Unlayer = %s", got)
+ }
+}
+
+func TestUnlayerKeepsSharedNodes(t *testing.T) {
+ x := &Node{ID: 3, NS: NSSymbol, Text: "x"}
+ f := &Node{ID: 2, NS: NSSymbol, Text: "f"}
+ g := &Node{ID: 5, NS: NSSymbol, Text: "g"}
+ a := &Node{ID: 1, NS: NSBuiltin, Text: ListNode}
+ b := &Node{ID: 4, NS: NSBuiltin, Text: ListNode}
+ l := Layers{
+ Nodes: [][]*Node{{a, b}, {f, x, g}},
+ Edges: [][]Edge{{{a, f}, {a, x}, {b, g}, {b, x}}, nil},
+ }
+ tree, err := Unlayer(l)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if tree.Forms[0].Children[1] != tree.Forms[1].Children[0] {
+ t.Error("x was duplicated instead of shared")
+ }
+}
+
+func TestUnlayerRejects(t *testing.T) {
+ node := func(id uint64, ns Namespace, text string) *Node { return &Node{ID: id, NS: ns, Text: text} }
+ list := func(id uint64) *Node { return node(id, NSBuiltin, ListNode) }
+ copyOf := func(n *Node) *Node { return node(n.ID, n.NS, n.Text) }
+
+ one := node(2, NSInt, "1")
+ oneCopy := copyOf(one)
+ stray := node(3, NSInt, "2")
+ root := list(1)
+
+ tests := []struct {
+ name string
+ layers Layers
+ want string
+ }{
+ {
+ "copy with a real child",
+ Layers{
+ Nodes: [][]*Node{{root}, {one}, {oneCopy}, {stray}},
+ Edges: [][]Edge{{{root, one}}, {{one, oneCopy}}, {{oneCopy, stray}}, nil},
+ },
+ "copy GOV.2.2.49 has real child GOV.2.3.50",
+ },
+ {
+ "atom with a real child",
+ Layers{
+ Nodes: [][]*Node{{root}, {one}, {stray}},
+ Edges: [][]Edge{{{root, one}}, {{one, stray}}, nil},
+ },
+ "atom GOV.2.2.49 has child GOV.2.3.50",
+ },
+ {
+ "list node with no elements",
+ Layers{Nodes: [][]*Node{{root}}, Edges: [][]Edge{nil}},
+ "list GOV.0.1.1 has no elements",
+ },
+ {
+ "list node whose only child is a copy",
+ func() Layers {
+ r, c := list(1), list(1)
+ return Layers{Nodes: [][]*Node{{r}, {c}}, Edges: [][]Edge{{{r, c}}, nil}}
+ }(),
+ "list GOV.0.1.1 has no elements",
+ },
+ {
+ ". first in a list",
+ func() Layers {
+ r, dot, a := list(1), node(2, NSBuiltin, Dot), node(3, NSSymbol, "a")
+ return Layers{Nodes: [][]*Node{{r}, {dot, a}}, Edges: [][]Edge{{{r, dot}, {r, a}}, nil}}
+ }(),
+ ". at position 1 of 2",
+ },
+ {
+ ". as a top-level form",
+ Layers{Nodes: [][]*Node{{node(1, NSBuiltin, Dot)}}, Edges: [][]Edge{nil}},
+ "depth 1: GOV.0.1.2 is outside a list",
+ },
+ {
+ "node with no parent",
+ Layers{
+ Nodes: [][]*Node{{root}, {one, stray}},
+ Edges: [][]Edge{{{root, one}}, nil},
+ },
+ "depth 2: GOV.2.3.50 has no parent",
+ },
+ {
+ "mapping that skips a depth",
+ Layers{
+ Nodes: [][]*Node{{root}, {one}, {stray}},
+ Edges: [][]Edge{{{root, one}, {root, stray}}, nil, nil},
+ },
+ "cert 1: mapping GOV.0.1.1 -> GOV.2.3.50 does not go from depth 1 to 2",
+ },
+ {
+ "two nodes with one id",
+ func() Layers {
+ r, a, b := list(1), node(2, NSSymbol, "a"), node(2, NSSymbol, "b")
+ return Layers{Nodes: [][]*Node{{r}, {a, b}}, Edges: [][]Edge{{{r, a}, {r, b}}, nil}}
+ }(),
+ "share node id 2",
+ },
+ {
+ "no forms",
+ Layers{},
+ "no top-level forms",
+ },
+ }
+ seen := map[string]string{}
+ for _, tt := range tests {
+ _, err := Unlayer(tt.layers)
+ if err == nil || !strings.Contains(err.Error(), tt.want) {
+ t.Errorf("%s: error = %v, want %q", tt.name, err, tt.want)
+ continue
+ }
+ // Cases that expect the same message are the same mistake; any other
+ // pair must be told apart.
+ if prev, dup := seen[err.Error()]; dup && prev != tt.want {
+ t.Errorf("%s gives the same error as a different case: %v", tt.name, err)
+ }
+ seen[err.Error()] = tt.want
+ }
+}