From 6f019acd7b7cd5e3f8b07d703abb11f32c97c2f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dirk=20St=C3=B6cker?= Date: Thu, 23 May 2024 15:44:12 +0200 Subject: [PATCH 1/3] support filtering --- cmd/certspotter/main.go | 30 +++++++++++++++ man/certspotter.md | 5 +++ monitor/config.go | 1 + monitor/discoveredcert.go | 7 +++- monitor/keylist.go | 77 +++++++++++++++++++++++++++++++++++++++ monitor/process.go | 2 + 6 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 monitor/keylist.go diff --git a/cmd/certspotter/main.go b/cmd/certspotter/main.go index 7779457..e0633f0 100644 --- a/cmd/certspotter/main.go +++ b/cmd/certspotter/main.go @@ -99,6 +99,16 @@ func defaultWatchListPathIfExists() string { return "" } } +func defaultKeyListPath() string { + return filepath.Join(defaultConfigDir(), "keylist") +} +func defaultKeyListPathIfExists() string { + if fileExists(defaultKeyListPath()) { + return defaultKeyListPath() + } else { + return "" + } +} func defaultScriptDir() string { return filepath.Join(defaultConfigDir(), "hooks.d") } @@ -124,6 +134,15 @@ func readWatchListFile(filename string) (monitor.WatchList, error) { return monitor.ReadWatchList(file) } +func readKeyListFile(filename string) (monitor.KeyList, error) { + file, err := os.Open(filename) + if err != nil { + return nil, simplifyError(err) + } + defer file.Close() + return monitor.ReadKeyList(file) +} + func readEmailFile(filename string) ([]string, error) { file, err := os.Open(filename) if err != nil { @@ -166,6 +185,7 @@ func main() { verbose bool version bool watchlist string + keylist string } flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)") flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email)) @@ -179,6 +199,7 @@ func main() { flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose") flag.BoolVar(&flags.version, "version", false, "Print version and exit") flag.StringVar(&flags.watchlist, "watchlist", defaultWatchListPathIfExists(), "File containing domain names to watch") + flag.StringVar(&flags.keylist, "keylist", defaultKeyListPathIfExists(), "File containing known key information") flag.Parse() if flags.version { @@ -242,6 +263,15 @@ func main() { config.WatchList = watchlist } + if flags.keylist != "" { + keylist, err := readKeyListFile(flags.keylist) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: error reading keylist from %q: %s\n", programName, flags.keylist, err) + os.Exit(1) + } + config.KeyList = keylist + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/man/certspotter.md b/man/certspotter.md index 880ebfa..d687989 100644 --- a/man/certspotter.md +++ b/man/certspotter.md @@ -115,6 +115,11 @@ You can use Cert Spotter to detect: certspotter reads the watch list only when starting up, so you must restart certspotter if you change it. +-keylist *PATH* + +: File containing known key information, one per line. A line consist of a + identifier and the sha256 of the public key separated by semi colon + # NOTIFICATIONS When certspotter detects a certificate matching your watchlist, or encounters diff --git a/monitor/config.go b/monitor/config.go index 378e3d5..5ec12a7 100644 --- a/monitor/config.go +++ b/monitor/config.go @@ -18,6 +18,7 @@ type Config struct { State StateProvider StartAtEnd bool WatchList WatchList + KeyList KeyList Verbose bool HealthCheckInterval time.Duration } diff --git a/monitor/discoveredcert.go b/monitor/discoveredcert.go index 46fc7ba..f61a7ae 100644 --- a/monitor/discoveredcert.go +++ b/monitor/discoveredcert.go @@ -23,6 +23,7 @@ import ( type DiscoveredCert struct { WatchItem WatchItem + KeyItem KeyItem LogEntry *LogEntry Info *certspotter.CertInfo Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate @@ -172,5 +173,9 @@ func certNotificationText(cert *DiscoveredCert, paths *certPaths) string { } func certNotificationSummary(cert *DiscoveredCert) string { - return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem) + keytext := "" + if cert.KeyItem.String() != "" { + keytext = fmt.Sprintf(" (known as %s)", cert.KeyItem.String()) + } + return fmt.Sprintf("Certificate Discovered for %s%s", cert.WatchItem, keytext) } diff --git a/monitor/keylist.go b/monitor/keylist.go new file mode 100644 index 0000000..34004f2 --- /dev/null +++ b/monitor/keylist.go @@ -0,0 +1,77 @@ +// Copyright (C) 2016, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bufio" + "encoding/hex" + "crypto/sha256" + "fmt" + "io" + "software.sslmate.com/src/certspotter" + "strings" +) + +type KeyItem struct { + domain string + keyinfo string +} + +type KeyList []KeyItem + +func ParseKeyItem(str string) (KeyItem, error) { + fields := strings.Split(str, ";") + if len(fields) == 0 { + return KeyItem{}, fmt.Errorf("empty domain") + } + if len(fields) == 1 { + return KeyItem{}, fmt.Errorf("empty key info") + } + return KeyItem{ + domain: fields[0], + keyinfo: fields[1], + }, nil +} + +func ReadKeyList(reader io.Reader) (KeyList, error) { + items := make(KeyList, 0, 50) + scanner := bufio.NewScanner(reader) + lineNo := 0 + for scanner.Scan() { + line := scanner.Text() + lineNo++ + if line == "" || strings.HasPrefix(line, "#") { + continue + } + item, err := ParseKeyItem(line) + if err != nil { + return nil, fmt.Errorf("%w on line %d", err, lineNo) + } + items = append(items, item) + } + return items, scanner.Err() +} + +func (item KeyItem) String() string { + return item.domain +} + +func (list KeyList) Matches(certInfo *certspotter.CertInfo) (bool, KeyItem) { + + for _, item := range list { + // better would be to convert the input and compare the bytes (no uppercase/lower-case issue) + pub := sha256.Sum256(certInfo.TBS.PublicKey.FullBytes) + sum := hex.EncodeToString(pub[:]) + if sum == item.keyinfo { + return true, item + } + } + return false, KeyItem{} +} diff --git a/monitor/process.go b/monitor/process.go index c416d55..3bd822d 100644 --- a/monitor/process.go +++ b/monitor/process.go @@ -91,11 +91,13 @@ func processCertificate(ctx context.Context, config *Config, entry *LogEntry, ce if !matched { return nil } + matched, keyItem := config.KeyList.Matches(certInfo) cert := &DiscoveredCert{ WatchItem: watchItem, LogEntry: entry, Info: certInfo, + KeyItem: keyItem, Chain: chain, TBSSHA256: sha256.Sum256(certInfo.TBS.Raw), SHA256: sha256.Sum256(chain[0]), From 1b441660abe70dfd0201cec780407f3f50013f60 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Wed, 16 Oct 2024 08:23:22 -0400 Subject: [PATCH 2/3] merkletree: replace IsComplete with more useful ContainsFirstN --- merkletree/fragment.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/merkletree/fragment.go b/merkletree/fragment.go index 4d82502..a679c72 100644 --- a/merkletree/fragment.go +++ b/merkletree/fragment.go @@ -90,8 +90,9 @@ func (tree FragmentedCollapsedTree) Subtrees() []CollapsedTree { } } -func (tree FragmentedCollapsedTree) IsComplete(size uint64) bool { - return len(tree.subtrees) == 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size == size +// Return true iff the tree contains at least the first n nodes (without any gaps) +func (tree FragmentedCollapsedTree) ContainsFirstN(n uint64) bool { + return len(tree.subtrees) >= 1 && tree.subtrees[0].offset == 0 && tree.subtrees[0].size >= n } func (tree *FragmentedCollapsedTree) Init(subtrees []CollapsedTree) error { From 9483eb0d4c2993af02e7ac06c215df54b494fe7f Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Mon, 25 Nov 2024 08:09:57 -0500 Subject: [PATCH 3/3] Add log list support for static-ct-api logs --- cmd/submitct/main.go | 11 ++++++----- loglist/helpers.go | 4 ++++ loglist/schema.go | 34 ++++++++++++++++++++++++++++++---- loglist/validate.go | 14 +++++++++++++- 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/cmd/submitct/main.go b/cmd/submitct/main.go index a5e8e0c..986da32 100644 --- a/cmd/submitct/main.go +++ b/cmd/submitct/main.go @@ -158,18 +158,19 @@ func main() { var logs []Log for _, ctlog := range list.AllLogs() { + submissionURL := ctlog.GetSubmissionURL() pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key) if err != nil { - log.Fatalf("%s: Failed to parse log public key: %s", ctlog.URL, err) + log.Fatalf("%s: Failed to parse log public key: %s", submissionURL, err) } verifier, err := ct.NewSignatureVerifier(pubkey) if err != nil { - log.Fatalf("%s: Failed to create signature verifier for log: %s", ctlog.URL, err) + log.Fatalf("%s: Failed to create signature verifier for log: %s", submissionURL, err) } logs = append(logs, Log{ Log: ctlog, SignatureVerifier: verifier, - LogClient: client.New(strings.TrimRight(ctlog.URL, "/")), + LogClient: client.New(strings.TrimRight(submissionURL, "/")), }) } @@ -212,11 +213,11 @@ func main() { go func(fingerprint [32]byte, ctlog Log) { sct, err := ctlog.SubmitChain(chain) if err != nil { - log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.URL, err) + log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.GetSubmissionURL(), err) atomic.AddUint32(&submitErrors, 1) } else if *verbose { timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000) - log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.URL, timestamp) + log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.GetSubmissionURL(), timestamp) } wg.Done() }(fingerprint, ctlog) diff --git a/loglist/helpers.go b/loglist/helpers.go index d236322..ab1586b 100644 --- a/loglist/helpers.go +++ b/loglist/helpers.go @@ -13,12 +13,16 @@ import ( "time" ) +// Return all tiled and non-tiled logs from all operators func (list *List) AllLogs() []*Log { logs := []*Log{} for operator := range list.Operators { for log := range list.Operators[operator].Logs { logs = append(logs, &list.Operators[operator].Logs[log]) } + for log := range list.Operators[operator].TiledLogs { + logs = append(logs, &list.Operators[operator].TiledLogs[log]) + } } return logs } diff --git a/loglist/schema.go b/loglist/schema.go index eb56c7d..7d070aa 100644 --- a/loglist/schema.go +++ b/loglist/schema.go @@ -22,16 +22,19 @@ type List struct { } type Operator struct { - Name string `json:"name"` - Email []string `json:"email"` - Logs []Log `json:"logs"` + Name string `json:"name"` + Email []string `json:"email"` + Logs []Log `json:"logs"` + TiledLogs []Log `json:"tiled_logs"` } type Log struct { Key []byte `json:"key"` LogID ct.SHA256Hash `json:"log_id"` MMD int `json:"mmd"` - URL string `json:"url"` + URL string `json:"url,omitempty"` // only for rfc6962 logs + SubmissionURL string `json:"submission_url,omitempty"` // only for static-ct-api logs + MonitoringURL string `json:"monitoring_url,omitempty"` // only for static-ct-api logs Description string `json:"description"` State State `json:"state"` DNS string `json:"dns"` @@ -44,6 +47,29 @@ type Log struct { // TODO: add previous_operators } +func (log *Log) IsRFC6962() bool { return log.URL != "" } +func (log *Log) IsStaticCTAPI() bool { return log.SubmissionURL != "" && log.MonitoringURL != "" } + +// Return URL prefix for submission using the RFC6962 protocol +func (log *Log) GetSubmissionURL() string { + if log.SubmissionURL != "" { + return log.SubmissionURL + } else { + return log.URL + } +} + +// Return URL prefix for monitoring. +// Since the protocol understood by the URL might be either RFC6962 or static-ct-api, this URL is +// only useful for informational purposes. +func (log *Log) GetMonitoringURL() string { + if log.MonitoringURL != "" { + return log.MonitoringURL + } else { + return log.URL + } +} + type State struct { Pending *struct { Timestamp time.Time `json:"timestamp"` diff --git a/loglist/validate.go b/loglist/validate.go index 65da8e4..8cc2750 100644 --- a/loglist/validate.go +++ b/loglist/validate.go @@ -26,7 +26,12 @@ func (list *List) Validate() error { func (operator *Operator) Validate() error { for i := range operator.Logs { if err := operator.Logs[i].Validate(); err != nil { - return fmt.Errorf("problem with %dth log (%s): %w", i, operator.Logs[i].LogIDString(), err) + return fmt.Errorf("problem with %dth non-tiled log (%s): %w", i, operator.Logs[i].LogIDString(), err) + } + } + for i := range operator.TiledLogs { + if err := operator.TiledLogs[i].Validate(); err != nil { + return fmt.Errorf("problem with %dth tiled log (%s): %w", i, operator.TiledLogs[i].LogIDString(), err) } } return nil @@ -37,5 +42,12 @@ func (log *Log) Validate() error { if log.LogID != realLogID { return fmt.Errorf("log ID does not match log key") } + + if !log.IsRFC6962() && !log.IsStaticCTAPI() { + return fmt.Errorf("URL(s) not provided") + } else if log.IsRFC6962() && log.IsStaticCTAPI() { + return fmt.Errorf("inconsistent URLs provided") + } + return nil }