From 06ce9370972508beede3b03d37dac740732ae70e Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Fri, 24 May 2024 09:08:17 -0400 Subject: [PATCH 1/9] Improve some comments --- monitor/state.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/monitor/state.go b/monitor/state.go index 6595988..48097e4 100644 --- a/monitor/state.go +++ b/monitor/state.go @@ -61,7 +61,8 @@ type StateProvider interface { // feailure is not associated with a log. NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error - // Called when an error occurs. The log is nil if the error is - // not associated with a log. Note that most errors are transient. + // Called when a non-fatal error occurs. The log is nil if the error is + // not associated with a log. Note that most errors are transient, and + // certspotter will retry the failed operation later. NotifyError(context.Context, *loglist.Log, error) error } From 7f17992c9cdc8c711fb733faf2315a24a993e5b2 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Sat, 25 May 2024 10:52:54 -0400 Subject: [PATCH 2/9] merkletree: factor out common initialization code --- merkletree/collapsed_tree.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/merkletree/collapsed_tree.go b/merkletree/collapsed_tree.go index f3dcd25..19a5a70 100644 --- a/merkletree/collapsed_tree.go +++ b/merkletree/collapsed_tree.go @@ -30,10 +30,11 @@ func EmptyCollapsedTree() *CollapsedTree { } func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) { - if len(nodes) != calculateNumNodes(size) { - return nil, fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes)) + tree := new(CollapsedTree) + if err := tree.Init(nodes, size); err != nil { + return nil, err } - return &CollapsedTree{nodes: nodes, size: size}, nil + return tree, nil } func CloneCollapsedTree(source *CollapsedTree) *CollapsedTree { @@ -105,10 +106,17 @@ func (tree *CollapsedTree) UnmarshalJSON(b []byte) error { if err := json.Unmarshal(b, &rawTree); err != nil { return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) } - if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) { - return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: nodes has wrong length (should be %d, not %d)", calculateNumNodes(rawTree.Size), len(rawTree.Nodes)) + if err := tree.Init(rawTree.Nodes, rawTree.Size); err != nil { + return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) } - tree.size = rawTree.Size - tree.nodes = rawTree.Nodes + return nil +} + +func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error { + if len(nodes) != calculateNumNodes(size) { + return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes)) + } + tree.size = size + tree.nodes = nodes return nil } From cc98a06bcb5daf2f0e0adfb131c09bcb6e2ba1c3 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Sat, 25 May 2024 11:19:55 -0400 Subject: [PATCH 3/9] merkletree: add method for getting collapsed tree nodes --- merkletree/collapsed_tree.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/merkletree/collapsed_tree.go b/merkletree/collapsed_tree.go index 19a5a70..7523481 100644 --- a/merkletree/collapsed_tree.go +++ b/merkletree/collapsed_tree.go @@ -87,6 +87,10 @@ func (tree *CollapsedTree) CalculateRoot() Hash { return hash } +func (tree *CollapsedTree) Nodes() []Hash { + return tree.nodes +} + func (tree *CollapsedTree) Size() uint64 { return tree.size } From 759631f7e6d73158aa0f6b5aa26ab871edc93234 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Sun, 9 Jun 2024 11:13:16 -0400 Subject: [PATCH 4/9] merkletree.Append: fix appending to empty trees --- merkletree/collapsed_tree.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/merkletree/collapsed_tree.go b/merkletree/collapsed_tree.go index 7523481..09eadb7 100644 --- a/merkletree/collapsed_tree.go +++ b/merkletree/collapsed_tree.go @@ -54,9 +54,11 @@ func (tree *CollapsedTree) Add(hash Hash) { } func (tree *CollapsedTree) Append(other *CollapsedTree) error { - maxSize := uint64(1) << bits.TrailingZeros64(tree.size) - if other.size > maxSize { - return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize) + if tree.size > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(tree.size) + if other.size > maxSize { + return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize) + } } tree.nodes = append(tree.nodes, other.nodes...) From b711c8762efa585629e45826d1c9857a0ef2aaf3 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Wed, 12 Jun 2024 11:21:58 -0400 Subject: [PATCH 5/9] Refine the CollapsedTree API --- merkletree/collapsed_tree.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/merkletree/collapsed_tree.go b/merkletree/collapsed_tree.go index 09eadb7..ce60c96 100644 --- a/merkletree/collapsed_tree.go +++ b/merkletree/collapsed_tree.go @@ -25,10 +25,12 @@ func calculateNumNodes(size uint64) int { return bits.OnesCount64(size) } +// TODO: phase out this function func EmptyCollapsedTree() *CollapsedTree { return &CollapsedTree{nodes: []Hash{}, size: 0} } +// TODO: phase out this function func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) { tree := new(CollapsedTree) if err := tree.Init(nodes, size); err != nil { @@ -37,23 +39,24 @@ func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) { return tree, nil } -func CloneCollapsedTree(source *CollapsedTree) *CollapsedTree { - nodes := make([]Hash, len(source.nodes)) - copy(nodes, source.nodes) - return &CollapsedTree{nodes: nodes, size: source.size} -} - func (tree CollapsedTree) Equal(other CollapsedTree) bool { return tree.size == other.size && slices.Equal(tree.nodes, other.nodes) } +func (tree CollapsedTree) Clone() CollapsedTree { + return CollapsedTree{ + nodes: slices.Clone(tree.nodes), + size: tree.size, + } +} + func (tree *CollapsedTree) Add(hash Hash) { tree.nodes = append(tree.nodes, hash) tree.size++ tree.collapse() } -func (tree *CollapsedTree) Append(other *CollapsedTree) error { +func (tree *CollapsedTree) Append(other CollapsedTree) error { if tree.size > 0 { maxSize := uint64(1) << bits.TrailingZeros64(tree.size) if other.size > maxSize { From fca2b8f8f13890a6558587e7445016dd1d567f81 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Thu, 13 Jun 2024 09:23:12 -0400 Subject: [PATCH 6/9] Add offset to merkletree.CollapsedTree so that it can represent arbitrary subtrees --- merkletree/collapsed_tree.go | 87 ++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 18 deletions(-) diff --git a/merkletree/collapsed_tree.go b/merkletree/collapsed_tree.go index ce60c96..9217b1f 100644 --- a/merkletree/collapsed_tree.go +++ b/merkletree/collapsed_tree.go @@ -16,9 +16,12 @@ import ( "slices" ) +// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending +// nodes and calculating the root hash. type CollapsedTree struct { - nodes []Hash - size uint64 + offset uint64 + nodes []Hash + size uint64 } func calculateNumNodes(size uint64) int { @@ -40,23 +43,44 @@ func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) { } func (tree CollapsedTree) Equal(other CollapsedTree) bool { - return tree.size == other.size && slices.Equal(tree.nodes, other.nodes) + return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes) } func (tree CollapsedTree) Clone() CollapsedTree { return CollapsedTree{ - nodes: slices.Clone(tree.nodes), - size: tree.size, + offset: tree.offset, + nodes: slices.Clone(tree.nodes), + size: tree.size, } } -func (tree *CollapsedTree) Add(hash Hash) { +// Add a new leaf hash to the end of the tree. +// Returns an error if and only if the new tree would be too large for the subtree offset. +// Always returns a nil error if tree.Offset() == 0. +func (tree *CollapsedTree) Add(hash Hash) error { + if tree.offset > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(tree.offset) + if tree.size+1 > maxSize { + return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize) + } + } tree.nodes = append(tree.nodes, hash) tree.size++ tree.collapse() + return nil } func (tree *CollapsedTree) Append(other CollapsedTree) error { + if tree.offset+tree.size != other.offset { + return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size) + } + if tree.offset > 0 { + newSize := tree.size + other.size + maxSize := uint64(1) << bits.TrailingZeros64(tree.offset) + if newSize > maxSize { + return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset) + } + } if tree.size > 0 { maxSize := uint64(1) << bits.TrailingZeros64(tree.size) if other.size > maxSize { @@ -79,7 +103,7 @@ func (tree *CollapsedTree) collapse() { } } -func (tree *CollapsedTree) CalculateRoot() Hash { +func (tree CollapsedTree) CalculateRoot() Hash { if len(tree.nodes) == 0 { return HashNothing() } @@ -92,30 +116,46 @@ func (tree *CollapsedTree) CalculateRoot() Hash { return hash } -func (tree *CollapsedTree) Nodes() []Hash { - return tree.nodes +// Return the subtree offset (0 if this represents an entire tree) +func (tree CollapsedTree) Offset() uint64 { + return tree.offset } -func (tree *CollapsedTree) Size() uint64 { +// Return a non-nil slice containing the nodes. The slice +// must not be modified. +func (tree CollapsedTree) Nodes() []Hash { + if tree.nodes == nil { + return []Hash{} + } else { + return tree.nodes + } +} + +// Return the number of leaf nodes in the tree. +func (tree CollapsedTree) Size() uint64 { return tree.size } +type collapsedTreeMessage struct { + Offset uint64 `json:"offset,omitempty"` + Nodes []Hash `json:"nodes"` // never nil + Size uint64 `json:"size"` +} + func (tree CollapsedTree) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]interface{}{ - "nodes": tree.nodes, - "size": tree.size, + return json.Marshal(collapsedTreeMessage{ + Offset: tree.offset, + Nodes: tree.Nodes(), + Size: tree.size, }) } func (tree *CollapsedTree) UnmarshalJSON(b []byte) error { - var rawTree struct { - Nodes []Hash `json:"nodes"` - Size uint64 `json:"size"` - } + var rawTree collapsedTreeMessage if err := json.Unmarshal(b, &rawTree); err != nil { return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) } - if err := tree.Init(rawTree.Nodes, rawTree.Size); err != nil { + if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil { return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) } return nil @@ -129,3 +169,14 @@ func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error { tree.nodes = nodes return nil } + +func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error { + if offset > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(offset) + if size > maxSize { + return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize) + } + } + tree.offset = offset + return tree.Init(nodes, size) +} From e570923ef2f128568140b8c8dd633afcf57a5f1c Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Thu, 13 Jun 2024 09:24:17 -0400 Subject: [PATCH 7/9] Add merkletree.FragmentedCollapsedTree --- merkletree/fragment.go | 120 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 merkletree/fragment.go diff --git a/merkletree/fragment.go b/merkletree/fragment.go new file mode 100644 index 0000000..686e668 --- /dev/null +++ b/merkletree/fragment.go @@ -0,0 +1,120 @@ +// Copyright (C) 2024 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package merkletree + +import ( + "encoding/json" + "fmt" + "slices" +) + +// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees +type FragmentedCollapsedTree struct { + subtrees []CollapsedTree // sorted by offset +} + +func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error { + return tree.Add(CollapsedTree{ + offset: position, + nodes: []Hash{hash}, + size: 1, + }) +} + +func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error { + if subtree.size == 0 { + return nil + } + i := len(tree.subtrees) + for i > 0 && tree.subtrees[i-1].offset > subtree.offset { + i-- + } + if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset { + return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size) + } + if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset { + return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset) + } + if i == 0 || tree.subtrees[i-1].Append(subtree) != nil { + tree.subtrees = slices.Insert(tree.subtrees, i, subtree) + i++ + } + for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil { + tree.subtrees = slices.Delete(tree.subtrees, i, i+1) + } + return nil +} + +func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error { + for _, subtree := range other.subtrees { + if err := tree.Add(subtree); err != nil { + return err + } + } + return nil +} + +func (tree *FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) { + var prevEnd uint64 + for i := range tree.subtrees { + if prevEnd != tree.subtrees[i].offset { + if !yield(prevEnd, tree.subtrees[i].offset) { + return + } + } + prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size + } + yield(prevEnd, 0) +} + +func (tree *FragmentedCollapsedTree) NumSubtrees() int { + return len(tree.subtrees) +} + +func (tree *FragmentedCollapsedTree) Subtree(i int) CollapsedTree { + return tree.subtrees[i] +} + +func (tree *FragmentedCollapsedTree) Subtrees() []CollapsedTree { + if tree.subtrees == nil { + return []CollapsedTree{} + } else { + return tree.subtrees + } +} + +func (tree *FragmentedCollapsedTree) IsComplete(size uint64) bool { + return len(tree.subtrees) == 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size == size +} + +func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error { + for i := 1; i < len(subtrees); i++ { + if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset { + return fmt.Errorf("subtrees %d and %d overlap", i-1, i) + } + } + tree.subtrees = subtrees + return nil +} + +func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) { + return json.Marshal(tree.Subtrees()) +} + +func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error { + var subtrees []CollapsedTree + if err := json.Unmarshal(b, &subtrees); err != nil { + return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err) + } + if err := tree.Init(subtrees); err != nil { + return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err) + } + return nil +} From 1b9a21baa85aa19f14e0c6b3835857455ad4f89d Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Thu, 13 Jun 2024 14:37:02 -0400 Subject: [PATCH 8/9] Remove unnecessary pointer receivers from FragmentedCollapsedTree --- merkletree/fragment.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/merkletree/fragment.go b/merkletree/fragment.go index 686e668..4d82502 100644 --- a/merkletree/fragment.go +++ b/merkletree/fragment.go @@ -61,7 +61,7 @@ func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error return nil } -func (tree *FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) { +func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) { var prevEnd uint64 for i := range tree.subtrees { if prevEnd != tree.subtrees[i].offset { @@ -74,15 +74,15 @@ func (tree *FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) { yield(prevEnd, 0) } -func (tree *FragmentedCollapsedTree) NumSubtrees() int { +func (tree FragmentedCollapsedTree) NumSubtrees() int { return len(tree.subtrees) } -func (tree *FragmentedCollapsedTree) Subtree(i int) CollapsedTree { +func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree { return tree.subtrees[i] } -func (tree *FragmentedCollapsedTree) Subtrees() []CollapsedTree { +func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree { if tree.subtrees == nil { return []CollapsedTree{} } else { @@ -90,7 +90,7 @@ func (tree *FragmentedCollapsedTree) Subtrees() []CollapsedTree { } } -func (tree *FragmentedCollapsedTree) IsComplete(size uint64) bool { +func (tree FragmentedCollapsedTree) IsComplete(size uint64) bool { return len(tree.subtrees) == 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size == size } From ed9ee59e8edd70b8496047b536535b3ce8b31a90 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Fri, 14 Jun 2024 15:16:26 -0400 Subject: [PATCH 9/9] Emphasize that start_at_end applies to new logs --- README.md | 2 +- cmd/certspotter/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b54d902..90ce140 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Cert Spotter requires Go version 1.19 or higher. 4. Configure your system to run `certspotter` as a daemon. You may want to specify the `-start_at_end` command line option to tell certspotter to start monitoring - logs at the end instead of the beginning. This saves significant bandwidth, but + new logs at the end instead of the beginning. This saves significant bandwidth, but you won't be notified about certificates which were logged before you started using certspotter. diff --git a/cmd/certspotter/main.go b/cmd/certspotter/main.go index 7779457..8d5e1c8 100644 --- a/cmd/certspotter/main.go +++ b/cmd/certspotter/main.go @@ -173,7 +173,7 @@ func main() { flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor") flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory") flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered") - flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)") + flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)") flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates") flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout") flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")