diff --git a/cmd/certspotter/main.go b/cmd/certspotter/main.go index 9dea831..7779457 100644 --- a/cmd/certspotter/main.go +++ b/cmd/certspotter/main.go @@ -190,33 +190,36 @@ func main() { os.Exit(2) } + fsstate := &monitor.FilesystemState{ + StateDir: flags.stateDir, + SaveCerts: !flags.noSave, + Script: flags.script, + ScriptDir: defaultScriptDir(), + Email: flags.email, + Stdout: flags.stdout, + } config := &monitor.Config{ LogListSource: flags.logs, - StateDir: flags.stateDir, - SaveCerts: !flags.noSave, + State: fsstate, StartAtEnd: flags.startAtEnd, Verbose: flags.verbose, - Script: flags.script, - ScriptDir: defaultScriptDir(), - Email: flags.email, - Stdout: flags.stdout, HealthCheckInterval: flags.healthcheck, } emailFileExists := false if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil { emailFileExists = true - config.Email = append(config.Email, emailRecipients...) + fsstate.Email = append(fsstate.Email, emailRecipients...) } else if !errors.Is(err, fs.ErrNotExist) { fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err) os.Exit(1) } - if len(config.Email) == 0 && !emailFileExists && config.Script == "" && !fileExists(config.ScriptDir) && config.Stdout == false { + if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false { fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName) fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n") fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile()) - fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", config.ScriptDir) + fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir) fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n") fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n") fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n") diff --git a/monitor/config.go b/monitor/config.go index 1e0d60c..378e3d5 100644 --- a/monitor/config.go +++ b/monitor/config.go @@ -15,14 +15,9 @@ import ( type Config struct { LogListSource string - StateDir string + State StateProvider StartAtEnd bool WatchList WatchList Verbose bool - SaveCerts bool - Script string - ScriptDir string - Email []string - Stdout bool HealthCheckInterval time.Duration } diff --git a/monitor/daemon.go b/monitor/daemon.go index 3a854a6..a824200 100644 --- a/monitor/daemon.go +++ b/monitor/daemon.go @@ -16,7 +16,6 @@ import ( "golang.org/x/sync/errgroup" "log" insecurerand "math/rand" - "path/filepath" "software.sslmate.com/src/certspotter/loglist" "time" ) @@ -51,18 +50,13 @@ type daemon struct { func (daemon *daemon) healthCheck(ctx context.Context) error { if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval { - textPath := filepath.Join(daemon.config.StateDir, "healthchecks", healthCheckFilename()) - event := &staleLogListEvent{ + info := &StaleLogListInfo{ Source: daemon.config.LogListSource, LastSuccess: daemon.logsLoadedAt, LastError: daemon.logListError, LastErrorTime: daemon.logListErrorAt, - TextPath: textPath, } - if err := event.save(); err != nil { - return fmt.Errorf("error saving stale log list event: %w", err) - } - if err := notify(ctx, daemon.config, event); err != nil { + if err := daemon.config.State.NotifyHealthCheckFailure(ctx, info); err != nil { return fmt.Errorf("error notifying about stale log list: %w", err) } } @@ -129,8 +123,8 @@ func (daemon *daemon) loadLogList(ctx context.Context) error { } func (daemon *daemon) run(ctx context.Context) error { - if err := prepareStateDir(daemon.config.StateDir); err != nil { - return fmt.Errorf("error preparing state directory: %w", err) + if err := daemon.config.State.Prepare(ctx); err != nil { + return fmt.Errorf("error preparing state: %w", err) } if err := daemon.loadLogList(ctx); err != nil { diff --git a/monitor/discoveredcert.go b/monitor/discoveredcert.go index f53b399..46fc7ba 100644 --- a/monitor/discoveredcert.go +++ b/monitor/discoveredcert.go @@ -21,21 +21,24 @@ import ( "software.sslmate.com/src/certspotter/ct" ) -type discoveredCert struct { +type DiscoveredCert struct { WatchItem WatchItem - LogEntry *logEntry + LogEntry *LogEntry Info *certspotter.CertInfo Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate TBSSHA256 [32]byte // computed over Info.TBS.Raw SHA256 [32]byte // computed over Chain[0] PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes Identifiers *certspotter.Identifiers - CertPath string // empty if not saved on the filesystem - JSONPath string // empty if not saved on the filesystem - TextPath string // empty if not saved on the filesystem } -func (cert *discoveredCert) pemChain() []byte { +type certPaths struct { + certPath string + jsonPath string + textPath string +} + +func (cert *DiscoveredCert) pemChain() []byte { var buffer bytes.Buffer for _, certBytes := range cert.Chain { if err := pem.Encode(&buffer, &pem.Block{ @@ -48,7 +51,7 @@ func (cert *discoveredCert) pemChain() []byte { return buffer.Bytes() } -func (cert *discoveredCert) json() any { +func (cert *DiscoveredCert) json() any { object := map[string]any{ "tbs_sha256": hex.EncodeToString(cert.TBSSHA256[:]), "pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]), @@ -67,23 +70,23 @@ func (cert *discoveredCert) json() any { return object } -func (cert *discoveredCert) save() error { - if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil { +func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error { + if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil { return err } - if err := writeJSONFile(cert.JSONPath, cert.json(), 0666); err != nil { + if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil { return err } - if err := writeTextFile(cert.TextPath, cert.Text(), 0666); err != nil { + if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil { return err } return nil } -func (cert *discoveredCert) Environ() []string { +func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string { env := []string{ "EVENT=discovered_cert", - "SUMMARY=" + cert.Summary(), + "SUMMARY=" + certNotificationSummary(cert), "CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented "LOG_URI=" + cert.LogEntry.Log.URL, "ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index), @@ -93,9 +96,12 @@ func (cert *discoveredCert) Environ() []string { "FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented "PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]), "PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented - "CERT_FILENAME=" + cert.CertPath, - "JSON_FILENAME=" + cert.JSONPath, - "TEXT_FILENAME=" + cert.TextPath, + } + + if paths != nil { + env = append(env, "CERT_FILENAME="+paths.certPath) + env = append(env, "JSON_FILENAME="+paths.jsonPath) + env = append(env, "TEXT_FILENAME="+paths.textPath) } if cert.Info.ValidityParseError == nil { @@ -130,7 +136,7 @@ func (cert *discoveredCert) Environ() []string { return env } -func (cert *discoveredCert) Text() string { +func certNotificationText(cert *DiscoveredCert, paths *certPaths) string { // TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration) text := new(strings.Builder) @@ -158,13 +164,13 @@ func (cert *discoveredCert) Text() string { } writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL)) writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:])) - if cert.CertPath != "" { - writeField("Filename", cert.CertPath) + if paths != nil { + writeField("Filename", paths.certPath) } return text.String() } -func (cert *discoveredCert) Summary() string { +func certNotificationSummary(cert *DiscoveredCert) string { return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem) } diff --git a/monitor/fsstate.go b/monitor/fsstate.go new file mode 100644 index 0000000..9b5e05d --- /dev/null +++ b/monitor/fsstate.go @@ -0,0 +1,228 @@ +// Copyright (C) 2024 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "software.sslmate.com/src/certspotter/ct" +) + +type FilesystemState struct { + StateDir string + SaveCerts bool + Script string + ScriptDir string + Email []string + Stdout bool +} + +func (s *FilesystemState) logStateDir(logID LogID) string { + return filepath.Join(s.StateDir, "logs", logID.Base64URLString()) +} + +func (s *FilesystemState) Prepare(ctx context.Context) error { + return prepareStateDir(s.StateDir) +} + +func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error { + var ( + stateDirPath = s.logStateDir(logID) + sthsDirPath = filepath.Join(stateDirPath, "unverified_sths") + malformedDirPath = filepath.Join(stateDirPath, "malformed_entries") + healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks") + ) + for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} { + if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + return nil +} + +func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) { + filePath := filepath.Join(s.logStateDir(logID), "state.json") + fileBytes, err := os.ReadFile(filePath) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } else if err != nil { + return nil, err + } + state := new(LogState) + if err := json.Unmarshal(fileBytes, state); err != nil { + return nil, fmt.Errorf("error parsing %s: %w", filePath, err) + } + return state, nil +} + +func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error { + filePath := filepath.Join(s.logStateDir(logID), "state.json") + return writeJSONFile(filePath, state, 0666) +} + +func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error { + sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") + return storeSTHInDir(sthsDirPath, sth) +} + +func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) { + sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") + return loadSTHsFromDir(sthsDirPath) +} + +func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error { + sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") + return removeSTHFromDir(sthsDirPath, sth) +} + +func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error { + var notifiedPath string + var paths *certPaths + if s.SaveCerts { + hexFingerprint := hex.EncodeToString(cert.SHA256[:]) + prefixPath := filepath.Join(s.StateDir, "certs", hexFingerprint[0:2]) + var ( + notifiedFilename = "." + hexFingerprint + ".notified" + certFilename = hexFingerprint + ".pem" + jsonFilename = hexFingerprint + ".v1.json" + textFilename = hexFingerprint + ".txt" + legacyCertFilename = hexFingerprint + ".cert.pem" + legacyPrecertFilename = hexFingerprint + ".precert.pem" + ) + + for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} { + if fileExists(filepath.Join(prefixPath, filename)) { + return nil + } + } + + if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err) + } + + notifiedPath = filepath.Join(prefixPath, notifiedFilename) + paths = &certPaths{ + certPath: filepath.Join(prefixPath, certFilename), + jsonPath: filepath.Join(prefixPath, jsonFilename), + textPath: filepath.Join(prefixPath, textFilename), + } + if err := writeCertFiles(cert, paths); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) + } + } else { + // TODO-4: save cert to temporary files, and defer their unlinking + } + + if err := s.notify(ctx, ¬ification{ + summary: certNotificationSummary(cert), + environ: certNotificationEnviron(cert, paths), + text: certNotificationText(cert, paths), + }); err != nil { + return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err) + } + + if notifiedPath != "" { + if err := os.WriteFile(notifiedPath, nil, 0666); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) + } + } + + return nil +} + +func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError string) error { + var ( + dirPath = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries") + entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index)) + textPath = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index)) + ) + + summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL) + + entryJSON := struct { + LeafInput []byte `json:"leaf_input"` + ExtraData []byte `json:"extra_data"` + }{ + LeafInput: entry.LeafInput, + ExtraData: entry.ExtraData, + } + + text := new(strings.Builder) + writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } + fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n") + writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL)) + writeField("Leaf Hash", entry.LeafHash.Base64String()) + writeField("Error", parseError) + + if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil { + return fmt.Errorf("error saving JSON file: %w", err) + } + if err := writeTextFile(textPath, text.String(), 0666); err != nil { + return fmt.Errorf("error saving texT file: %w", err) + } + + environ := []string{ + "EVENT=malformed_cert", + "SUMMARY=" + summary, + "LOG_URI=" + entry.Log.URL, + "ENTRY_INDEX=" + fmt.Sprint(entry.Index), + "LEAF_HASH=" + entry.LeafHash.Base64String(), + "PARSE_ERROR=" + parseError, + "ENTRY_FILENAME=" + entryPath, + "TEXT_FILENAME=" + textPath, + "CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented + } + + if err := s.notify(ctx, ¬ification{ + environ: environ, + summary: summary, + text: text.String(), + }); err != nil { + return err + } + return nil +} + +func (s *FilesystemState) healthCheckTextPath(info HealthCheckFailure) string { + if ctlog := info.Log(); ctlog == nil { + return filepath.Join(s.StateDir, "healthchecks", healthCheckFilename()) + } else { + return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks", healthCheckFilename()) + } +} + +func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, info HealthCheckFailure) error { + textPath := s.healthCheckTextPath(info) + environ := []string{ + "EVENT=error", + "SUMMARY=" + info.Summary(), + "TEXT_FILENAME=" + textPath, + } + text := info.Text() + if err := writeTextFile(textPath, text, 0666); err != nil { + return fmt.Errorf("error saving text file: %w", err) + } + if err := s.notify(ctx, ¬ification{ + environ: environ, + summary: info.Summary(), + text: text, + }); err != nil { + return err + } + return nil +} diff --git a/monitor/healthcheck.go b/monitor/healthcheck.go index 51d12ee..98caaba 100644 --- a/monitor/healthcheck.go +++ b/monitor/healthcheck.go @@ -11,10 +11,7 @@ package monitor import ( "context" - "errors" "fmt" - "io/fs" - "path/filepath" "strings" "time" @@ -27,52 +24,38 @@ func healthCheckFilename() string { } func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error { - var ( - stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString()) - stateFilePath = filepath.Join(stateDirPath, "state.json") - sthsDirPath = filepath.Join(stateDirPath, "unverified_sths") - textPath = filepath.Join(stateDirPath, "healthchecks", healthCheckFilename()) - ) - state, err := loadStateFile(stateFilePath) - if errors.Is(err, fs.ErrNotExist) { + state, err := config.State.LoadLogState(ctx, ctlog.LogID) + if err != nil { + return fmt.Errorf("error loading log state: %w", err) + } else if state == nil { return nil - } else if err != nil { - return fmt.Errorf("error loading state file: %w", err) } if time.Since(state.LastSuccess) < config.HealthCheckInterval { return nil } - sths, err := loadSTHsFromDir(sthsDirPath) + sths, err := config.State.LoadSTHs(ctx, ctlog.LogID) if err != nil { - return fmt.Errorf("error loading STHs directory: %w", err) + return fmt.Errorf("error loading STHs: %w", err) } if len(sths) == 0 { - event := &staleSTHEvent{ - Log: ctlog, + info := &StaleSTHInfo{ + log: ctlog, LastSuccess: state.LastSuccess, LatestSTH: state.VerifiedSTH, - TextPath: textPath, } - if err := event.save(); err != nil { - return fmt.Errorf("error saving stale STH event: %w", err) - } - if err := notify(ctx, config, event); err != nil { + if err := config.State.NotifyHealthCheckFailure(ctx, info); err != nil { return fmt.Errorf("error notifying about stale STH: %w", err) } } else { - event := &backlogEvent{ - Log: ctlog, + info := &BacklogInfo{ + log: ctlog, LatestSTH: sths[len(sths)-1], Position: state.DownloadPosition.Size(), - TextPath: textPath, } - if err := event.save(); err != nil { - return fmt.Errorf("error saving backlog event: %w", err) - } - if err := notify(ctx, config, event); err != nil { + if err := config.State.NotifyHealthCheckFailure(ctx, info); err != nil { return fmt.Errorf("error notifying about backlog: %w", err) } } @@ -80,65 +63,58 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err return nil } -type staleSTHEvent struct { - Log *loglist.Log +type HealthCheckFailure interface { + Summary() string + Text() string + Log() *loglist.Log // returns nil if failure is not associated with a log +} + +type StaleSTHInfo struct { + log *loglist.Log LastSuccess time.Time LatestSTH *ct.SignedTreeHead // may be nil - TextPath string } -type backlogEvent struct { - Log *loglist.Log + +type BacklogInfo struct { + log *loglist.Log LatestSTH *ct.SignedTreeHead Position uint64 - TextPath string } -type staleLogListEvent struct { + +type StaleLogListInfo struct { Source string LastSuccess time.Time LastError string LastErrorTime time.Time - TextPath string } -func (e *backlogEvent) Backlog() uint64 { +func (e *BacklogInfo) Backlog() uint64 { return e.LatestSTH.TreeSize - e.Position } -func (e *staleSTHEvent) Environ() []string { - return []string{ - "EVENT=error", - "SUMMARY=" + e.Summary(), - "TEXT_FILENAME=" + e.TextPath, - } +func (e *StaleSTHInfo) Log() *loglist.Log { + return e.log } -func (e *backlogEvent) Environ() []string { - return []string{ - "EVENT=error", - "SUMMARY=" + e.Summary(), - "TEXT_FILENAME=" + e.TextPath, - } +func (e *BacklogInfo) Log() *loglist.Log { + return e.log } -func (e *staleLogListEvent) Environ() []string { - return []string{ - "EVENT=error", - "SUMMARY=" + e.Summary(), - "TEXT_FILENAME=" + e.TextPath, - } +func (e *StaleLogListInfo) Log() *loglist.Log { + return nil } -func (e *staleSTHEvent) Summary() string { - return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess) +func (e *StaleSTHInfo) Summary() string { + return fmt.Sprintf("Unable to contact %s since %s", e.log.URL, e.LastSuccess) } -func (e *backlogEvent) Summary() string { - return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL) +func (e *BacklogInfo) Summary() string { + return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.log.URL) } -func (e *staleLogListEvent) Summary() string { +func (e *StaleLogListInfo) Summary() string { return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess) } -func (e *staleSTHEvent) Text() string { +func (e *StaleSTHInfo) Text() string { text := new(strings.Builder) - fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess) + fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.log.URL, e.LastSuccess) fmt.Fprintf(text, "\n") fmt.Fprintf(text, "For details, see certspotter's stderr output.\n") fmt.Fprintf(text, "\n") @@ -149,9 +125,9 @@ func (e *staleSTHEvent) Text() string { } return text.String() } -func (e *backlogEvent) Text() string { +func (e *BacklogInfo) Text() string { text := new(strings.Builder) - fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL) + fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.log.URL) fmt.Fprintf(text, "\n") fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n") fmt.Fprintf(text, "\n") @@ -160,7 +136,7 @@ func (e *backlogEvent) Text() string { fmt.Fprintf(text, " Backlog = %d\n", e.Backlog()) return text.String() } -func (e *staleLogListEvent) Text() string { +func (e *StaleLogListInfo) Text() string { text := new(strings.Builder) fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess) fmt.Fprintf(text, "\n") @@ -170,14 +146,4 @@ func (e *staleLogListEvent) Text() string { return text.String() } -func (e *staleSTHEvent) save() error { - return writeTextFile(e.TextPath, e.Text(), 0666) -} -func (e *backlogEvent) save() error { - return writeTextFile(e.TextPath, e.Text(), 0666) -} -func (e *staleLogListEvent) save() error { - return writeTextFile(e.TextPath, e.Text(), 0666) -} - // TODO-3: make the errors more actionable diff --git a/monitor/malformed.go b/monitor/malformed.go deleted file mode 100644 index 35cac00..0000000 --- a/monitor/malformed.go +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (C) 2023 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package monitor - -import ( - "fmt" - "strings" -) - -type malformedLogEntry struct { - Entry *logEntry - Error string - EntryPath string - TextPath string -} - -func (malformed *malformedLogEntry) entryJSON() any { - return struct { - LeafInput []byte `json:"leaf_input"` - ExtraData []byte `json:"extra_data"` - }{ - LeafInput: malformed.Entry.LeafInput, - ExtraData: malformed.Entry.ExtraData, - } -} - -func (malformed *malformedLogEntry) save() error { - if err := writeJSONFile(malformed.EntryPath, malformed.entryJSON(), 0666); err != nil { - return err - } - if err := writeTextFile(malformed.TextPath, malformed.Text(), 0666); err != nil { - return err - } - return nil -} - -func (malformed *malformedLogEntry) Environ() []string { - return []string{ - "EVENT=malformed_cert", - "SUMMARY=" + malformed.Summary(), - "LOG_URI=" + malformed.Entry.Log.URL, - "ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index), - "LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(), - "PARSE_ERROR=" + malformed.Error, - "ENTRY_FILENAME=" + malformed.EntryPath, - "TEXT_FILENAME=" + malformed.TextPath, - "CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented - } -} - -func (malformed *malformedLogEntry) Text() string { - text := new(strings.Builder) - writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } - - fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n") - writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL)) - writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String()) - writeField("Error", malformed.Error) - - return text.String() -} - -func (malformed *malformedLogEntry) Summary() string { - return fmt.Sprintf("Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL) -} diff --git a/monitor/monitor.go b/monitor/monitor.go index a97013d..3c65670 100644 --- a/monitor/monitor.go +++ b/monitor/monitor.go @@ -14,10 +14,7 @@ import ( "crypto/x509" "errors" "fmt" - "io/fs" "log" - "os" - "path/filepath" "strings" "time" @@ -73,17 +70,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie ctx, cancel := context.WithCancel(ctx) defer cancel() - var ( - stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString()) - stateFilePath = filepath.Join(stateDirPath, "state.json") - sthsDirPath = filepath.Join(stateDirPath, "unverified_sths") - malformedDirPath = filepath.Join(stateDirPath, "malformed_entries") - healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks") - ) - for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} { - if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { - return fmt.Errorf("error creating state directory: %w", err) - } + if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil { + return fmt.Errorf("error preparing state: %w", err) } startTime := time.Now() @@ -95,12 +83,15 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie return nil } latestSTH.LogID = ctlog.LogID - if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil { + if err := config.State.StoreSTH(ctx, ctlog.LogID, latestSTH); err != nil { return fmt.Errorf("error storing latest STH: %w", err) } - state, err := loadStateFile(stateFilePath) - if errors.Is(err, fs.ErrNotExist) { + state, err := config.State.LoadLogState(ctx, ctlog.LogID) + if err != nil { + return fmt.Errorf("error loading log state: %w", err) + } + if state == nil { if config.StartAtEnd { tree, err := reconstructTree(ctx, logClient, latestSTH) if isFatalLogError(err) { @@ -109,14 +100,14 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err)) return nil } - state = &stateFile{ + state = &LogState{ DownloadPosition: tree, VerifiedPosition: tree, VerifiedSTH: latestSTH, LastSuccess: startTime.UTC(), } } else { - state = &stateFile{ + state = &LogState{ DownloadPosition: merkletree.EmptyCollapsedTree(), VerifiedPosition: merkletree.EmptyCollapsedTree(), VerifiedSTH: nil, @@ -126,21 +117,19 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie if config.Verbose { log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size()) } - if err := state.store(stateFilePath); err != nil { - return fmt.Errorf("error storing state file: %w", err) + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) } - } else if err != nil { - return fmt.Errorf("error loading state file: %w", err) } - sths, err := loadSTHsFromDir(sthsDirPath) + sths, err := config.State.LoadSTHs(ctx, ctlog.LogID) if err != nil { - return fmt.Errorf("error loading STHs directory: %w", err) + return fmt.Errorf("error loading STHs: %w", err) } for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() { // TODO-4: audit sths[0] against state.VerifiedSTH - if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil { + if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil { return fmt.Errorf("error removing STH: %w", err) } sths = sths[1:] @@ -150,8 +139,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie if config.Verbose { log.Printf("saving state in defer for %s", ctlog.URL) } - if err := state.store(stateFilePath); err != nil && returnedErr == nil { - returnedErr = fmt.Errorf("error storing state file: %w", err) + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil { + returnedErr = fmt.Errorf("error storing log state: %w", err) } }() @@ -174,7 +163,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd) }() for rawEntry := range entries { - entry := &logEntry{ + entry := &LogEntry{ Log: ctlog, Index: state.DownloadPosition.Size(), LeafInput: rawEntry.LeafInput, @@ -194,8 +183,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash)) state.DownloadPosition = state.VerifiedPosition - if err := state.store(stateFilePath); err != nil { - return fmt.Errorf("error storing state file: %w", err) + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) } return nil } @@ -203,7 +192,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie state.VerifiedPosition = state.DownloadPosition state.VerifiedSTH = sths[0] shouldSaveState = true - if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil { + if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil { return fmt.Errorf("error removing verified STH: %w", err) } @@ -211,7 +200,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie } if shouldSaveState { - if err := state.store(stateFilePath); err != nil { + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { return fmt.Errorf("error storing state file: %w", err) } } diff --git a/monitor/notify.go b/monitor/notify.go index d59fbbc..d371927 100644 --- a/monitor/notify.go +++ b/monitor/notify.go @@ -25,31 +25,31 @@ import ( var stdoutMu sync.Mutex -type notification interface { - Environ() []string - Summary() string - Text() string +type notification struct { + environ []string + summary string + text string } -func notify(ctx context.Context, config *Config, notif notification) error { - if config.Stdout { +func (s *FilesystemState) notify(ctx context.Context, notif *notification) error { + if s.Stdout { writeToStdout(notif) } - if len(config.Email) > 0 { - if err := sendEmail(ctx, config.Email, notif); err != nil { + if len(s.Email) > 0 { + if err := sendEmail(ctx, s.Email, notif); err != nil { return err } } - if config.Script != "" { - if err := execScript(ctx, config.Script, notif); err != nil { + if s.Script != "" { + if err := execScript(ctx, s.Script, notif); err != nil { return err } } - if config.ScriptDir != "" { - if err := execScriptDir(ctx, config.ScriptDir, notif); err != nil { + if s.ScriptDir != "" { + if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil { return err } } @@ -57,25 +57,25 @@ func notify(ctx context.Context, config *Config, notif notification) error { return nil } -func writeToStdout(notif notification) { +func writeToStdout(notif *notification) { stdoutMu.Lock() defer stdoutMu.Unlock() - os.Stdout.WriteString(notif.Text() + "\n") + os.Stdout.WriteString(notif.text + "\n") } -func sendEmail(ctx context.Context, to []string, notif notification) error { +func sendEmail(ctx context.Context, to []string, notif *notification) error { stdin := new(bytes.Buffer) stderr := new(bytes.Buffer) fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", ")) - fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.Summary()) + fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary) fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat)) fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID()) fmt.Fprintf(stdin, "Mime-Version: 1.0\n") fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n") fmt.Fprintf(stdin, "X-Mailer: certspotter\n") fmt.Fprintf(stdin, "\n") - fmt.Fprint(stdin, notif.Text()) + fmt.Fprint(stdin, notif.text) args := []string{"-i", "--"} args = append(args, to...) @@ -95,12 +95,12 @@ func sendEmail(ctx context.Context, to []string, notif notification) error { } } -func execScript(ctx context.Context, scriptName string, notif notification) error { +func execScript(ctx context.Context, scriptName string, notif *notification) error { stderr := new(bytes.Buffer) cmd := exec.CommandContext(ctx, scriptName) cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, notif.Environ()...) + cmd.Env = append(cmd.Env, notif.environ...) cmd.Stderr = stderr if err := cmd.Run(); err == nil { @@ -116,7 +116,7 @@ func execScript(ctx context.Context, scriptName string, notif notification) erro } } -func execScriptDir(ctx context.Context, dirPath string, notif notification) error { +func execScriptDir(ctx context.Context, dirPath string, notif *notification) error { dirents, err := os.ReadDir(dirPath) if errors.Is(err, fs.ErrNotExist) { return nil diff --git a/monitor/process.go b/monitor/process.go index 6115d06..faa2531 100644 --- a/monitor/process.go +++ b/monitor/process.go @@ -13,19 +13,14 @@ import ( "bytes" "context" "crypto/sha256" - "encoding/hex" - "errors" "fmt" - "io/fs" - "os" - "path/filepath" "software.sslmate.com/src/certspotter" "software.sslmate.com/src/certspotter/ct" "software.sslmate.com/src/certspotter/loglist" "software.sslmate.com/src/certspotter/merkletree" ) -type logEntry struct { +type LogEntry struct { Log *loglist.Log Index uint64 LeafInput []byte @@ -33,7 +28,7 @@ type logEntry struct { LeafHash merkletree.Hash } -func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error { +func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error { leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput)) if err != nil { return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err)) @@ -48,7 +43,7 @@ func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error } } -func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error { +func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error { certInfo, err := certspotter.MakeCertInfoFromRawCert(cert) if err != nil { return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err)) @@ -69,7 +64,7 @@ func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, c return processCertificate(ctx, config, entry, certInfo, chain) } -func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error { +func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error { certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate) if err != nil { return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err)) @@ -87,7 +82,7 @@ func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry return processCertificate(ctx, config, entry, certInfo, chain) } -func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error { +func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error { identifiers, err := certInfo.ParseIdentifiers() if err != nil { return processMalformedLogEntry(ctx, config, entry, err) @@ -97,7 +92,7 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce return nil } - cert := &discoveredCert{ + cert := &DiscoveredCert{ WatchItem: watchItem, LogEntry: entry, Info: certInfo, @@ -108,68 +103,15 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce Identifiers: identifiers, } - var notifiedPath string - if config.SaveCerts { - hexFingerprint := hex.EncodeToString(cert.SHA256[:]) - prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2]) - var ( - notifiedFilename = "." + hexFingerprint + ".notified" - certFilename = hexFingerprint + ".pem" - jsonFilename = hexFingerprint + ".v1.json" - textFilename = hexFingerprint + ".txt" - legacyCertFilename = hexFingerprint + ".cert.pem" - legacyPrecertFilename = hexFingerprint + ".precert.pem" - ) - - for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} { - if fileExists(filepath.Join(prefixPath, filename)) { - return nil - } - } - - if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { - return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err) - } - - notifiedPath = filepath.Join(prefixPath, notifiedFilename) - cert.CertPath = filepath.Join(prefixPath, certFilename) - cert.JSONPath = filepath.Join(prefixPath, jsonFilename) - cert.TextPath = filepath.Join(prefixPath, textFilename) - - if err := cert.save(); err != nil { - return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) - } - } else { - // TODO-4: save cert to temporary files, and defer their unlinking - } - - if err := notify(ctx, config, cert); err != nil { - return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err) - } - - if notifiedPath != "" { - if err := os.WriteFile(notifiedPath, nil, 0666); err != nil { - return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err) - } + if err := config.State.NotifyCert(ctx, cert); err != nil { + return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err) } return nil } -func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error { - dirPath := filepath.Join(config.StateDir, "logs", entry.Log.LogID.Base64URLString(), "malformed_entries") - malformed := &malformedLogEntry{ - Entry: entry, - Error: parseError.Error(), - EntryPath: filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index)), - TextPath: filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index)), - } - - if err := malformed.save(); err != nil { - return fmt.Errorf("error saving malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err) - } - - if err := notify(ctx, config, malformed); err != nil { +func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error { + if err := config.State.NotifyMalformedEntry(ctx, entry, parseError.Error()); err != nil { return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err) } return nil diff --git a/monitor/state.go b/monitor/state.go new file mode 100644 index 0000000..c9ee9e4 --- /dev/null +++ b/monitor/state.go @@ -0,0 +1,61 @@ +// Copyright (C) 2024 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/merkletree" + "time" +) + +type LogState struct { + DownloadPosition *merkletree.CollapsedTree `json:"download_position"` + VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"` + VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"` + LastSuccess time.Time `json:"last_success"` +} + +type StateProvider interface { + // Initialize the state. Called before any other method in this interface. + // Idempotent: returns nil if the state is already initialized. + Prepare(context.Context) error + + // Initialize the state for the given log. Called before any other method + // with the log ID. Idempotent: returns nil if log state already initialized. + PrepareLog(context.Context, LogID) error + + // Store log state for retrieval by LoadLogState. + StoreLogState(context.Context, LogID, *LogState) error + + // Load log state that was previously stored with StoreLogState. + // Returns nil, nil if StoreLogState has not been called yet for this log. + LoadLogState(context.Context, LogID) (*LogState, error) + + // Store STH for retrieval by LoadSTHs. If an STH with the same + // timestamp and root hash is already stored, this STH can be ignored. + StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error + + // Load all STHs for this log previously stored with StoreSTH. + // The returned slice must be sorted by tree size. + LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error) + + // Remove an STH so it is no longer returned by LoadSTHs. + RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error + + // Called when a certificate matching the watch list is discovered. + NotifyCert(context.Context, *DiscoveredCert) error + + // Called when certspotter fails to parse a log entry. + NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError string) error + + // Called when a health check fails. + NotifyHealthCheckFailure(context.Context, HealthCheckFailure) error +} diff --git a/monitor/statedir.go b/monitor/statedir.go index e94927c..9e3f4cb 100644 --- a/monitor/statedir.go +++ b/monitor/statedir.go @@ -76,13 +76,13 @@ func migrateLogStateDirV1(dir string) error { return fmt.Errorf("error unmarshaling %s: %w", treePath, err) } - stateFile := stateFile{ + stateFile := LogState{ DownloadPosition: &tree, VerifiedPosition: &tree, VerifiedSTH: &sth, LastSuccess: time.Now().UTC(), } - if stateFile.store(filepath.Join(dir, "state.json")); err != nil { + if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil { return err } diff --git a/monitor/statefile.go b/monitor/statefile.go deleted file mode 100644 index dcefe60..0000000 --- a/monitor/statefile.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (C) 2023 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package monitor - -import ( - "encoding/json" - "fmt" - "os" - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/merkletree" - "time" -) - -type stateFile struct { - DownloadPosition *merkletree.CollapsedTree `json:"download_position"` - VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"` - VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"` - LastSuccess time.Time `json:"last_success"` -} - -func loadStateFile(filePath string) (*stateFile, error) { - fileBytes, err := os.ReadFile(filePath) - if err != nil { - return nil, err - } - file := new(stateFile) - if err := json.Unmarshal(fileBytes, file); err != nil { - return nil, fmt.Errorf("error parsing %s: %w", filePath, err) - } - return file, nil -} - -func (file *stateFile) store(filePath string) error { - return writeJSONFile(filePath, file, 0666) -}