diff --git a/README.md b/README.md index b54d902..90ce140 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Cert Spotter requires Go version 1.19 or higher. 4. Configure your system to run `certspotter` as a daemon. You may want to specify the `-start_at_end` command line option to tell certspotter to start monitoring - logs at the end instead of the beginning. This saves significant bandwidth, but + new logs at the end instead of the beginning. This saves significant bandwidth, but you won't be notified about certificates which were logged before you started using certspotter. diff --git a/cmd/certspotter/main.go b/cmd/certspotter/main.go index e0633f0..0d3ae3e 100644 --- a/cmd/certspotter/main.go +++ b/cmd/certspotter/main.go @@ -193,7 +193,7 @@ func main() { flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor") flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory") flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered") - flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)") + flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring new logs from the end rather than the beginning (saves considerable bandwidth)") flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates") flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout") flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose") diff --git a/merkletree/collapsed_tree.go b/merkletree/collapsed_tree.go index f3dcd25..9217b1f 100644 --- a/merkletree/collapsed_tree.go +++ b/merkletree/collapsed_tree.go @@ -16,46 +16,76 @@ import ( "slices" ) +// CollapsedTree is an efficient representation of a Merkle (sub)tree that permits appending +// nodes and calculating the root hash. type CollapsedTree struct { - nodes []Hash - size uint64 + offset uint64 + nodes []Hash + size uint64 } func calculateNumNodes(size uint64) int { return bits.OnesCount64(size) } +// TODO: phase out this function func EmptyCollapsedTree() *CollapsedTree { return &CollapsedTree{nodes: []Hash{}, size: 0} } +// TODO: phase out this function func NewCollapsedTree(nodes []Hash, size uint64) (*CollapsedTree, error) { - if len(nodes) != calculateNumNodes(size) { - return nil, fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes)) + tree := new(CollapsedTree) + if err := tree.Init(nodes, size); err != nil { + return nil, err } - return &CollapsedTree{nodes: nodes, size: size}, nil -} - -func CloneCollapsedTree(source *CollapsedTree) *CollapsedTree { - nodes := make([]Hash, len(source.nodes)) - copy(nodes, source.nodes) - return &CollapsedTree{nodes: nodes, size: source.size} + return tree, nil } func (tree CollapsedTree) Equal(other CollapsedTree) bool { - return tree.size == other.size && slices.Equal(tree.nodes, other.nodes) + return tree.offset == other.offset && tree.size == other.size && slices.Equal(tree.nodes, other.nodes) } -func (tree *CollapsedTree) Add(hash Hash) { +func (tree CollapsedTree) Clone() CollapsedTree { + return CollapsedTree{ + offset: tree.offset, + nodes: slices.Clone(tree.nodes), + size: tree.size, + } +} + +// Add a new leaf hash to the end of the tree. +// Returns an error if and only if the new tree would be too large for the subtree offset. +// Always returns a nil error if tree.Offset() == 0. +func (tree *CollapsedTree) Add(hash Hash) error { + if tree.offset > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(tree.offset) + if tree.size+1 > maxSize { + return fmt.Errorf("subtree at offset %d is already at maximum size %d", tree.offset, maxSize) + } + } tree.nodes = append(tree.nodes, hash) tree.size++ tree.collapse() + return nil } -func (tree *CollapsedTree) Append(other *CollapsedTree) error { - maxSize := uint64(1) << bits.TrailingZeros64(tree.size) - if other.size > maxSize { - return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize) +func (tree *CollapsedTree) Append(other CollapsedTree) error { + if tree.offset+tree.size != other.offset { + return fmt.Errorf("subtree at offset %d cannot be appended to subtree ending at offset %d", other.offset, tree.offset+tree.size) + } + if tree.offset > 0 { + newSize := tree.size + other.size + maxSize := uint64(1) << bits.TrailingZeros64(tree.offset) + if newSize > maxSize { + return fmt.Errorf("size of new subtree (%d) would exceed maximum size %d for a subtree at offset %d", newSize, maxSize, tree.offset) + } + } + if tree.size > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(tree.size) + if other.size > maxSize { + return fmt.Errorf("tree of size %d is too large to append to a tree of size %d (maximum size is %d)", other.size, tree.size, maxSize) + } } tree.nodes = append(tree.nodes, other.nodes...) @@ -73,7 +103,7 @@ func (tree *CollapsedTree) collapse() { } } -func (tree *CollapsedTree) CalculateRoot() Hash { +func (tree CollapsedTree) CalculateRoot() Hash { if len(tree.nodes) == 0 { return HashNothing() } @@ -86,29 +116,67 @@ func (tree *CollapsedTree) CalculateRoot() Hash { return hash } -func (tree *CollapsedTree) Size() uint64 { +// Return the subtree offset (0 if this represents an entire tree) +func (tree CollapsedTree) Offset() uint64 { + return tree.offset +} + +// Return a non-nil slice containing the nodes. The slice +// must not be modified. +func (tree CollapsedTree) Nodes() []Hash { + if tree.nodes == nil { + return []Hash{} + } else { + return tree.nodes + } +} + +// Return the number of leaf nodes in the tree. +func (tree CollapsedTree) Size() uint64 { return tree.size } +type collapsedTreeMessage struct { + Offset uint64 `json:"offset,omitempty"` + Nodes []Hash `json:"nodes"` // never nil + Size uint64 `json:"size"` +} + func (tree CollapsedTree) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]interface{}{ - "nodes": tree.nodes, - "size": tree.size, + return json.Marshal(collapsedTreeMessage{ + Offset: tree.offset, + Nodes: tree.Nodes(), + Size: tree.size, }) } func (tree *CollapsedTree) UnmarshalJSON(b []byte) error { - var rawTree struct { - Nodes []Hash `json:"nodes"` - Size uint64 `json:"size"` - } + var rawTree collapsedTreeMessage if err := json.Unmarshal(b, &rawTree); err != nil { return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) } - if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) { - return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: nodes has wrong length (should be %d, not %d)", calculateNumNodes(rawTree.Size), len(rawTree.Nodes)) + if err := tree.InitSubtree(rawTree.Offset, rawTree.Nodes, rawTree.Size); err != nil { + return fmt.Errorf("error unmarshalling Collapsed Merkle Tree: %w", err) } - tree.size = rawTree.Size - tree.nodes = rawTree.Nodes return nil } + +func (tree *CollapsedTree) Init(nodes []Hash, size uint64) error { + if len(nodes) != calculateNumNodes(size) { + return fmt.Errorf("nodes has wrong length (should be %d, not %d)", calculateNumNodes(size), len(nodes)) + } + tree.size = size + tree.nodes = nodes + return nil +} + +func (tree *CollapsedTree) InitSubtree(offset uint64, nodes []Hash, size uint64) error { + if offset > 0 { + maxSize := uint64(1) << bits.TrailingZeros64(offset) + if size > maxSize { + return fmt.Errorf("subtree size (%d) is too large for offset %d (maximum size is %d)", size, offset, maxSize) + } + } + tree.offset = offset + return tree.Init(nodes, size) +} diff --git a/merkletree/fragment.go b/merkletree/fragment.go new file mode 100644 index 0000000..4d82502 --- /dev/null +++ b/merkletree/fragment.go @@ -0,0 +1,120 @@ +// Copyright (C) 2024 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package merkletree + +import ( + "encoding/json" + "fmt" + "slices" +) + +// FragmentedCollapsedTree represents a sequence of non-overlapping subtrees +type FragmentedCollapsedTree struct { + subtrees []CollapsedTree // sorted by offset +} + +func (tree *FragmentedCollapsedTree) AddHash(position uint64, hash Hash) error { + return tree.Add(CollapsedTree{ + offset: position, + nodes: []Hash{hash}, + size: 1, + }) +} + +func (tree *FragmentedCollapsedTree) Add(subtree CollapsedTree) error { + if subtree.size == 0 { + return nil + } + i := len(tree.subtrees) + for i > 0 && tree.subtrees[i-1].offset > subtree.offset { + i-- + } + if i > 0 && tree.subtrees[i-1].offset+tree.subtrees[i-1].size > subtree.offset { + return fmt.Errorf("overlaps with subtree ending at %d", tree.subtrees[i-1].offset+tree.subtrees[i-1].size) + } + if i < len(tree.subtrees) && subtree.offset+subtree.size > tree.subtrees[i].offset { + return fmt.Errorf("overlaps with subtree starting at %d", tree.subtrees[i].offset) + } + if i == 0 || tree.subtrees[i-1].Append(subtree) != nil { + tree.subtrees = slices.Insert(tree.subtrees, i, subtree) + i++ + } + for i < len(tree.subtrees) && tree.subtrees[i-1].Append(tree.subtrees[i]) == nil { + tree.subtrees = slices.Delete(tree.subtrees, i, i+1) + } + return nil +} + +func (tree *FragmentedCollapsedTree) Merge(other FragmentedCollapsedTree) error { + for _, subtree := range other.subtrees { + if err := tree.Add(subtree); err != nil { + return err + } + } + return nil +} + +func (tree FragmentedCollapsedTree) Gaps(yield func(uint64, uint64) bool) { + var prevEnd uint64 + for i := range tree.subtrees { + if prevEnd != tree.subtrees[i].offset { + if !yield(prevEnd, tree.subtrees[i].offset) { + return + } + } + prevEnd = tree.subtrees[i].offset + tree.subtrees[i].size + } + yield(prevEnd, 0) +} + +func (tree FragmentedCollapsedTree) NumSubtrees() int { + return len(tree.subtrees) +} + +func (tree FragmentedCollapsedTree) Subtree(i int) CollapsedTree { + return tree.subtrees[i] +} + +func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree { + if tree.subtrees == nil { + return []CollapsedTree{} + } else { + return tree.subtrees + } +} + +func (tree FragmentedCollapsedTree) IsComplete(size uint64) bool { + return len(tree.subtrees) == 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size == size +} + +func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error { + for i := 1; i < len(subtrees); i++ { + if subtrees[i-1].offset+subtrees[i-1].size > subtrees[i].offset { + return fmt.Errorf("subtrees %d and %d overlap", i-1, i) + } + } + tree.subtrees = subtrees + return nil +} + +func (tree FragmentedCollapsedTree) MarshalJSON() ([]byte, error) { + return json.Marshal(tree.Subtrees()) +} + +func (tree *FragmentedCollapsedTree) UnmarshalJSON(b []byte) error { + var subtrees []CollapsedTree + if err := json.Unmarshal(b, &subtrees); err != nil { + return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err) + } + if err := tree.Init(subtrees); err != nil { + return fmt.Errorf("error unmarshaling Fragmented Collapsed Merkle Tree: %w", err) + } + return nil +} diff --git a/monitor/state.go b/monitor/state.go index 6595988..48097e4 100644 --- a/monitor/state.go +++ b/monitor/state.go @@ -61,7 +61,8 @@ type StateProvider interface { // feailure is not associated with a log. NotifyHealthCheckFailure(context.Context, *loglist.Log, HealthCheckFailure) error - // Called when an error occurs. The log is nil if the error is - // not associated with a log. Note that most errors are transient. + // Called when a non-fatal error occurs. The log is nil if the error is + // not associated with a log. Note that most errors are transient, and + // certspotter will retry the failed operation later. NotifyError(context.Context, *loglist.Log, error) error }