Tandem
/
certspotter
mirror of https://github.com/SSLMate/certspotter.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Abstract state storage and notification logic behind an interface

This commit is contained in:
Andrew Ayer 2024-04-03 20:06:00 -04:00
parent 740bf5ac55
commit 5e0737353c
13 changed files with 428 additions and 358 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/certspotter/main.go
View File

@ -190,33 +190,36 @@ func main() {
os.Exit(2)
}
config := &monitor.Config{
LogListSource: flags.logs,
fsstate := &monitor.FilesystemState{
StateDir: flags.stateDir,
SaveCerts: !flags.noSave,
StartAtEnd: flags.startAtEnd,
Verbose: flags.verbose,
Script: flags.script,
ScriptDir: defaultScriptDir(),
Email: flags.email,
Stdout: flags.stdout,
}
config := &monitor.Config{
LogListSource: flags.logs,
State: fsstate,
StartAtEnd: flags.startAtEnd,
Verbose: flags.verbose,
HealthCheckInterval: flags.healthcheck,
}
emailFileExists := false
if emailRecipients, err := readEmailFile(defaultEmailFile()); err == nil {
emailFileExists = true
config.Email = append(config.Email, emailRecipients...)
fsstate.Email = append(fsstate.Email, emailRecipients...)
} else if !errors.Is(err, fs.ErrNotExist) {
fmt.Fprintf(os.Stderr, "%s: error reading email recipients file %q: %s\n", programName, defaultEmailFile(), err)
os.Exit(1)
}
if len(config.Email) == 0 && !emailFileExists && config.Script == "" && !fileExists(config.ScriptDir) && config.Stdout == false {
if len(fsstate.Email) == 0 && !emailFileExists && fsstate.Script == "" && !fileExists(fsstate.ScriptDir) && fsstate.Stdout == false {
fmt.Fprintf(os.Stderr, "%s: no notification methods were specified\n", programName)
fmt.Fprintf(os.Stderr, "Please specify at least one of the following notification methods:\n")
fmt.Fprintf(os.Stderr, " - Place one or more email addresses in %s (one address per line)\n", defaultEmailFile())
fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", config.ScriptDir)
fmt.Fprintf(os.Stderr, " - Place one or more executable scripts in the %s directory\n", fsstate.ScriptDir)
fmt.Fprintf(os.Stderr, " - Specify an email address using the -email flag\n")
fmt.Fprintf(os.Stderr, " - Specify the path to an executable script using the -script flag\n")
fmt.Fprintf(os.Stderr, " - Specify the -stdout flag\n")

7
monitor/config.go
View File

@ -15,14 +15,9 @@ import (
type Config struct {
LogListSource string
StateDir string
State StateProvider
StartAtEnd bool
WatchList WatchList
Verbose bool
SaveCerts bool
Script string
ScriptDir string
Email []string
Stdout bool
HealthCheckInterval time.Duration
}

14
monitor/daemon.go
View File

@ -16,7 +16,6 @@ import (
"golang.org/x/sync/errgroup"
"log"
insecurerand "math/rand"
"path/filepath"
"software.sslmate.com/src/certspotter/loglist"
"time"
)
@ -51,18 +50,13 @@ type daemon struct {
func (daemon *daemon) healthCheck(ctx context.Context) error {
if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
textPath := filepath.Join(daemon.config.StateDir, "healthchecks", healthCheckFilename())
event := &staleLogListEvent{
info := &StaleLogListInfo{
Source: daemon.config.LogListSource,
LastSuccess: daemon.logsLoadedAt,
LastError: daemon.logListError,
LastErrorTime: daemon.logListErrorAt,
TextPath: textPath,
}
if err := event.save(); err != nil {
return fmt.Errorf("error saving stale log list event: %w", err)
}
if err := notify(ctx, daemon.config, event); err != nil {
if err := daemon.config.State.NotifyHealthCheckFailure(ctx, info); err != nil {
return fmt.Errorf("error notifying about stale log list: %w", err)
}
}
@ -129,8 +123,8 @@ func (daemon *daemon) loadLogList(ctx context.Context) error {
}
func (daemon *daemon) run(ctx context.Context) error {
if err := prepareStateDir(daemon.config.StateDir); err != nil {
return fmt.Errorf("error preparing state directory: %w", err)
if err := daemon.config.State.Prepare(ctx); err != nil {
return fmt.Errorf("error preparing state: %w", err)
}
if err := daemon.loadLogList(ctx); err != nil {

46
monitor/discoveredcert.go
View File

@ -21,21 +21,24 @@ import (
"software.sslmate.com/src/certspotter/ct"
)
type discoveredCert struct {
type DiscoveredCert struct {
WatchItem WatchItem
LogEntry *logEntry
LogEntry *LogEntry
Info *certspotter.CertInfo
Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate
TBSSHA256 [32]byte // computed over Info.TBS.Raw
SHA256 [32]byte // computed over Chain[0]
PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes
Identifiers *certspotter.Identifiers
CertPath string // empty if not saved on the filesystem
JSONPath string // empty if not saved on the filesystem
TextPath string // empty if not saved on the filesystem
}
func (cert *discoveredCert) pemChain() []byte {
type certPaths struct {
certPath string
jsonPath string
textPath string
}
func (cert *DiscoveredCert) pemChain() []byte {
var buffer bytes.Buffer
for _, certBytes := range cert.Chain {
if err := pem.Encode(&buffer, &pem.Block{
@ -48,7 +51,7 @@ func (cert *discoveredCert) pemChain() []byte {
return buffer.Bytes()
}
func (cert *discoveredCert) json() any {
func (cert *DiscoveredCert) json() any {
object := map[string]any{
"tbs_sha256": hex.EncodeToString(cert.TBSSHA256[:]),
"pubkey_sha256": hex.EncodeToString(cert.PubkeySHA256[:]),
@ -67,23 +70,23 @@ func (cert *discoveredCert) json() any {
return object
}
func (cert *discoveredCert) save() error {
if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil {
func writeCertFiles(cert *DiscoveredCert, paths *certPaths) error {
if err := writeFile(paths.certPath, cert.pemChain(), 0666); err != nil {
return err
}
if err := writeJSONFile(cert.JSONPath, cert.json(), 0666); err != nil {
if err := writeJSONFile(paths.jsonPath, cert.json(), 0666); err != nil {
return err
}
if err := writeTextFile(cert.TextPath, cert.Text(), 0666); err != nil {
if err := writeTextFile(paths.textPath, certNotificationText(cert, paths), 0666); err != nil {
return err
}
return nil
}
func (cert *discoveredCert) Environ() []string {
func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string {
env := []string{
"EVENT=discovered_cert",
"SUMMARY=" + cert.Summary(),
"SUMMARY=" + certNotificationSummary(cert),
"CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented
"LOG_URI=" + cert.LogEntry.Log.URL,
"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
@ -93,9 +96,12 @@ func (cert *discoveredCert) Environ() []string {
"FINGERPRINT=" + hex.EncodeToString(cert.SHA256[:]), // backwards compat with pre-0.15.0; not documented
"PUBKEY_SHA256=" + hex.EncodeToString(cert.PubkeySHA256[:]),
"PUBKEY_HASH=" + hex.EncodeToString(cert.PubkeySHA256[:]), // backwards compat with pre-0.15.0; not documented
"CERT_FILENAME=" + cert.CertPath,
"JSON_FILENAME=" + cert.JSONPath,
"TEXT_FILENAME=" + cert.TextPath,
}
if paths != nil {
env = append(env, "CERT_FILENAME="+paths.certPath)
env = append(env, "JSON_FILENAME="+paths.jsonPath)
env = append(env, "TEXT_FILENAME="+paths.textPath)
}
if cert.Info.ValidityParseError == nil {
@ -130,7 +136,7 @@ func (cert *discoveredCert) Environ() []string {
return env
}
func (cert *discoveredCert) Text() string {
func certNotificationText(cert *DiscoveredCert, paths *certPaths) string {
// TODO-4: improve the output: include WatchItem, indicate hash algorithm used for fingerprints, ... (look at SSLMate email for inspiration)
text := new(strings.Builder)
@ -158,13 +164,13 @@ func (cert *discoveredCert) Text() string {
}
writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL))
writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:]))
if cert.CertPath != "" {
writeField("Filename", cert.CertPath)
if paths != nil {
writeField("Filename", paths.certPath)
}
return text.String()
}
func (cert *discoveredCert) Summary() string {
func certNotificationSummary(cert *DiscoveredCert) string {
return fmt.Sprintf("Certificate Discovered for %s", cert.WatchItem)
}

228
monitor/fsstate.go Normal file
View File

@ -0,0 +1,228 @@
// Copyright (C) 2024 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"software.sslmate.com/src/certspotter/ct"
)
type FilesystemState struct {
StateDir string
SaveCerts bool
Script string
ScriptDir string
Email []string
Stdout bool
}
func (s *FilesystemState) logStateDir(logID LogID) string {
return filepath.Join(s.StateDir, "logs", logID.Base64URLString())
}
func (s *FilesystemState) Prepare(ctx context.Context) error {
return prepareStateDir(s.StateDir)
}
func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
var (
stateDirPath = s.logStateDir(logID)
sthsDirPath = filepath.Join(stateDirPath, "unverified_sths")
malformedDirPath = filepath.Join(stateDirPath, "malformed_entries")
healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
)
for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return err
}
}
return nil
}
func (s *FilesystemState) LoadLogState(ctx context.Context, logID LogID) (*LogState, error) {
filePath := filepath.Join(s.logStateDir(logID), "state.json")
fileBytes, err := os.ReadFile(filePath)
if errors.Is(err, fs.ErrNotExist) {
return nil, nil
} else if err != nil {
return nil, err
}
state := new(LogState)
if err := json.Unmarshal(fileBytes, state); err != nil {
return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
}
return state, nil
}
func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state *LogState) error {
filePath := filepath.Join(s.logStateDir(logID), "state.json")
return writeJSONFile(filePath, state, 0666)
}
func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
return storeSTHInDir(sthsDirPath, sth)
}
func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) {
sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
return loadSTHsFromDir(sthsDirPath)
}
func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error {
sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths")
return removeSTHFromDir(sthsDirPath, sth)
}
func (s *FilesystemState) NotifyCert(ctx context.Context, cert *DiscoveredCert) error {
var notifiedPath string
var paths *certPaths
if s.SaveCerts {
hexFingerprint := hex.EncodeToString(cert.SHA256[:])
prefixPath := filepath.Join(s.StateDir, "certs", hexFingerprint[0:2])
var (
notifiedFilename = "." + hexFingerprint + ".notified"
certFilename = hexFingerprint + ".pem"
jsonFilename = hexFingerprint + ".v1.json"
textFilename = hexFingerprint + ".txt"
legacyCertFilename = hexFingerprint + ".cert.pem"
legacyPrecertFilename = hexFingerprint + ".precert.pem"
)
for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
if fileExists(filepath.Join(prefixPath, filename)) {
return nil
}
}
if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
}
notifiedPath = filepath.Join(prefixPath, notifiedFilename)
paths = &certPaths{
certPath: filepath.Join(prefixPath, certFilename),
jsonPath: filepath.Join(prefixPath, jsonFilename),
textPath: filepath.Join(prefixPath, textFilename),
}
if err := writeCertFiles(cert, paths); err != nil {
return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
}
} else {
// TODO-4: save cert to temporary files, and defer their unlinking
}
if err := s.notify(ctx, &notification{
summary: certNotificationSummary(cert),
environ: certNotificationEnviron(cert, paths),
text: certNotificationText(cert, paths),
}); err != nil {
return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
}
if notifiedPath != "" {
if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
}
}
return nil
}
func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError string) error {
var (
dirPath = filepath.Join(s.logStateDir(entry.Log.LogID), "malformed_entries")
entryPath = filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index))
textPath = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index))
)
summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL)
entryJSON := struct {
LeafInput []byte `json:"leaf_input"`
ExtraData []byte `json:"extra_data"`
}{
LeafInput: entry.LeafInput,
ExtraData: entry.ExtraData,
}
text := new(strings.Builder)
writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL))
writeField("Leaf Hash", entry.LeafHash.Base64String())
writeField("Error", parseError)
if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil {
return fmt.Errorf("error saving JSON file: %w", err)
}
if err := writeTextFile(textPath, text.String(), 0666); err != nil {
return fmt.Errorf("error saving texT file: %w", err)
}
environ := []string{
"EVENT=malformed_cert",
"SUMMARY=" + summary,
"LOG_URI=" + entry.Log.URL,
"ENTRY_INDEX=" + fmt.Sprint(entry.Index),
"LEAF_HASH=" + entry.LeafHash.Base64String(),
"PARSE_ERROR=" + parseError,
"ENTRY_FILENAME=" + entryPath,
"TEXT_FILENAME=" + textPath,
"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
}
if err := s.notify(ctx, &notification{
environ: environ,
summary: summary,
text: text.String(),
}); err != nil {
return err
}
return nil
}
func (s *FilesystemState) healthCheckTextPath(info HealthCheckFailure) string {
if ctlog := info.Log(); ctlog == nil {
return filepath.Join(s.StateDir, "healthchecks", healthCheckFilename())
} else {
return filepath.Join(s.logStateDir(ctlog.LogID), "healthchecks", healthCheckFilename())
}
}
func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, info HealthCheckFailure) error {
textPath := s.healthCheckTextPath(info)
environ := []string{
"EVENT=error",
"SUMMARY=" + info.Summary(),
"TEXT_FILENAME=" + textPath,
}
text := info.Text()
if err := writeTextFile(textPath, text, 0666); err != nil {
return fmt.Errorf("error saving text file: %w", err)
}
if err := s.notify(ctx, &notification{
environ: environ,
summary: info.Summary(),
text: text,
}); err != nil {
return err
}
return nil
}

118
monitor/healthcheck.go
View File

@ -11,10 +11,7 @@ package monitor
import (
"context"
"errors"
"fmt"
"io/fs"
"path/filepath"
"strings"
"time"
@ -27,52 +24,38 @@ func healthCheckFilename() string {
}
func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) error {
var (
stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
stateFilePath = filepath.Join(stateDirPath, "state.json")
sthsDirPath = filepath.Join(stateDirPath, "unverified_sths")
textPath = filepath.Join(stateDirPath, "healthchecks", healthCheckFilename())
)
state, err := loadStateFile(stateFilePath)
if errors.Is(err, fs.ErrNotExist) {
state, err := config.State.LoadLogState(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading log state: %w", err)
} else if state == nil {
return nil
} else if err != nil {
return fmt.Errorf("error loading state file: %w", err)
}
if time.Since(state.LastSuccess) < config.HealthCheckInterval {
return nil
}
sths, err := loadSTHsFromDir(sthsDirPath)
sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading STHs directory: %w", err)
return fmt.Errorf("error loading STHs: %w", err)
}
if len(sths) == 0 {
event := &staleSTHEvent{
Log: ctlog,
info := &StaleSTHInfo{
log: ctlog,
LastSuccess: state.LastSuccess,
LatestSTH: state.VerifiedSTH,
TextPath: textPath,
}
if err := event.save(); err != nil {
return fmt.Errorf("error saving stale STH event: %w", err)
}
if err := notify(ctx, config, event); err != nil {
if err := config.State.NotifyHealthCheckFailure(ctx, info); err != nil {
return fmt.Errorf("error notifying about stale STH: %w", err)
}
} else {
event := &backlogEvent{
Log: ctlog,
info := &BacklogInfo{
log: ctlog,
LatestSTH: sths[len(sths)-1],
Position: state.DownloadPosition.Size(),
TextPath: textPath,
}
if err := event.save(); err != nil {
return fmt.Errorf("error saving backlog event: %w", err)
}
if err := notify(ctx, config, event); err != nil {
if err := config.State.NotifyHealthCheckFailure(ctx, info); err != nil {
return fmt.Errorf("error notifying about backlog: %w", err)
}
}
@ -80,65 +63,58 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
return nil
}
type staleSTHEvent struct {
Log *loglist.Log
type HealthCheckFailure interface {
Summary() string
Text() string
Log() *loglist.Log // returns nil if failure is not associated with a log
}
type StaleSTHInfo struct {
log *loglist.Log
LastSuccess time.Time
LatestSTH *ct.SignedTreeHead // may be nil
TextPath string
}
type backlogEvent struct {
Log *loglist.Log
type BacklogInfo struct {
log *loglist.Log
LatestSTH *ct.SignedTreeHead
Position uint64
TextPath string
}
type staleLogListEvent struct {
type StaleLogListInfo struct {
Source string
LastSuccess time.Time
LastError string
LastErrorTime time.Time
TextPath string
}
func (e *backlogEvent) Backlog() uint64 {
func (e *BacklogInfo) Backlog() uint64 {
return e.LatestSTH.TreeSize - e.Position
}
func (e *staleSTHEvent) Environ() []string {
return []string{
"EVENT=error",
"SUMMARY=" + e.Summary(),
"TEXT_FILENAME=" + e.TextPath,
func (e *StaleSTHInfo) Log() *loglist.Log {
return e.log
}
func (e *BacklogInfo) Log() *loglist.Log {
return e.log
}
func (e *backlogEvent) Environ() []string {
return []string{
"EVENT=error",
"SUMMARY=" + e.Summary(),
"TEXT_FILENAME=" + e.TextPath,
}
}
func (e *staleLogListEvent) Environ() []string {
return []string{
"EVENT=error",
"SUMMARY=" + e.Summary(),
"TEXT_FILENAME=" + e.TextPath,
}
func (e *StaleLogListInfo) Log() *loglist.Log {
return nil
}
func (e *staleSTHEvent) Summary() string {
return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
func (e *StaleSTHInfo) Summary() string {
return fmt.Sprintf("Unable to contact %s since %s", e.log.URL, e.LastSuccess)
}
func (e *backlogEvent) Summary() string {
return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL)
func (e *BacklogInfo) Summary() string {
return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.log.URL)
}
func (e *staleLogListEvent) Summary() string {
func (e *StaleLogListInfo) Summary() string {
return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess)
}
func (e *staleSTHEvent) Text() string {
func (e *StaleSTHInfo) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.log.URL, e.LastSuccess)
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")
@ -149,9 +125,9 @@ func (e *staleSTHEvent) Text() string {
}
return text.String()
}
func (e *backlogEvent) Text() string {
func (e *BacklogInfo) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.log.URL)
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")
@ -160,7 +136,7 @@ func (e *backlogEvent) Text() string {
fmt.Fprintf(text, " Backlog = %d\n", e.Backlog())
return text.String()
}
func (e *staleLogListEvent) Text() string {
func (e *StaleLogListInfo) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
fmt.Fprintf(text, "\n")
@ -170,14 +146,4 @@ func (e *staleLogListEvent) Text() string {
return text.String()
}
func (e *staleSTHEvent) save() error {
return writeTextFile(e.TextPath, e.Text(), 0666)
}
func (e *backlogEvent) save() error {
return writeTextFile(e.TextPath, e.Text(), 0666)
}
func (e *staleLogListEvent) save() error {
return writeTextFile(e.TextPath, e.Text(), 0666)
}
// TODO-3: make the errors more actionable

72
monitor/malformed.go
View File

@ -1,72 +0,0 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"fmt"
"strings"
)
type malformedLogEntry struct {
Entry *logEntry
Error string
EntryPath string
TextPath string
}
func (malformed *malformedLogEntry) entryJSON() any {
return struct {
LeafInput []byte `json:"leaf_input"`
ExtraData []byte `json:"extra_data"`
}{
LeafInput: malformed.Entry.LeafInput,
ExtraData: malformed.Entry.ExtraData,
}
}
func (malformed *malformedLogEntry) save() error {
if err := writeJSONFile(malformed.EntryPath, malformed.entryJSON(), 0666); err != nil {
return err
}
if err := writeTextFile(malformed.TextPath, malformed.Text(), 0666); err != nil {
return err
}
return nil
}
func (malformed *malformedLogEntry) Environ() []string {
return []string{
"EVENT=malformed_cert",
"SUMMARY=" + malformed.Summary(),
"LOG_URI=" + malformed.Entry.Log.URL,
"ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index),
"LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(),
"PARSE_ERROR=" + malformed.Error,
"ENTRY_FILENAME=" + malformed.EntryPath,
"TEXT_FILENAME=" + malformed.TextPath,
"CERT_PARSEABLE=no", // backwards compat with pre-0.15.0; not documented
}
}
func (malformed *malformedLogEntry) Text() string {
text := new(strings.Builder)
writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL))
writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String())
writeField("Error", malformed.Error)
return text.String()
}
func (malformed *malformedLogEntry) Summary() string {
return fmt.Sprintf("Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL)
}

55
monitor/monitor.go
View File

@ -14,10 +14,7 @@ import (
"crypto/x509"
"errors"
"fmt"
"io/fs"
"log"
"os"
"path/filepath"
"strings"
"time"
@ -73,17 +70,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var (
stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
stateFilePath = filepath.Join(stateDirPath, "state.json")
sthsDirPath = filepath.Join(stateDirPath, "unverified_sths")
malformedDirPath = filepath.Join(stateDirPath, "malformed_entries")
healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
)
for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("error creating state directory: %w", err)
}
if err := config.State.PrepareLog(ctx, ctlog.LogID); err != nil {
return fmt.Errorf("error preparing state: %w", err)
}
startTime := time.Now()
@ -95,12 +83,15 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
return nil
}
latestSTH.LogID = ctlog.LogID
if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil {
if err := config.State.StoreSTH(ctx, ctlog.LogID, latestSTH); err != nil {
return fmt.Errorf("error storing latest STH: %w", err)
}
state, err := loadStateFile(stateFilePath)
if errors.Is(err, fs.ErrNotExist) {
state, err := config.State.LoadLogState(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading log state: %w", err)
}
if state == nil {
if config.StartAtEnd {
tree, err := reconstructTree(ctx, logClient, latestSTH)
if isFatalLogError(err) {
@ -109,14 +100,14 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err))
return nil
}
state = &stateFile{
state = &LogState{
DownloadPosition: tree,
VerifiedPosition: tree,
VerifiedSTH: latestSTH,
LastSuccess: startTime.UTC(),
}
} else {
state = &stateFile{
state = &LogState{
DownloadPosition: merkletree.EmptyCollapsedTree(),
VerifiedPosition: merkletree.EmptyCollapsedTree(),
VerifiedSTH: nil,
@ -126,21 +117,19 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
if config.Verbose {
log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
}
if err := state.store(stateFilePath); err != nil {
return fmt.Errorf("error storing state file: %w", err)
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
} else if err != nil {
return fmt.Errorf("error loading state file: %w", err)
}
sths, err := loadSTHsFromDir(sthsDirPath)
sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
if err != nil {
return fmt.Errorf("error loading STHs directory: %w", err)
return fmt.Errorf("error loading STHs: %w", err)
}
for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
// TODO-4: audit sths[0] against state.VerifiedSTH
if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
return fmt.Errorf("error removing STH: %w", err)
}
sths = sths[1:]
@ -150,8 +139,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
if config.Verbose {
log.Printf("saving state in defer for %s", ctlog.URL)
}
if err := state.store(stateFilePath); err != nil && returnedErr == nil {
returnedErr = fmt.Errorf("error storing state file: %w", err)
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil {
returnedErr = fmt.Errorf("error storing log state: %w", err)
}
}()
@ -174,7 +163,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
}()
for rawEntry := range entries {
entry := &logEntry{
entry := &LogEntry{
Log: ctlog,
Index: state.DownloadPosition.Size(),
LeafInput: rawEntry.LeafInput,
@ -194,8 +183,8 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
state.DownloadPosition = state.VerifiedPosition
if err := state.store(stateFilePath); err != nil {
return fmt.Errorf("error storing state file: %w", err)
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing log state: %w", err)
}
return nil
}
@ -203,7 +192,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
state.VerifiedPosition = state.DownloadPosition
state.VerifiedSTH = sths[0]
shouldSaveState = true
if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil {
return fmt.Errorf("error removing verified STH: %w", err)
}
@ -211,7 +200,7 @@ func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClie
}
if shouldSaveState {
if err := state.store(stateFilePath); err != nil {
if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {
return fmt.Errorf("error storing state file: %w", err)
}
}

40
monitor/notify.go
View File

@ -25,31 +25,31 @@ import (
var stdoutMu sync.Mutex
type notification interface {
Environ() []string
Summary() string
Text() string
type notification struct {
environ []string
summary string
text string
}
func notify(ctx context.Context, config *Config, notif notification) error {
if config.Stdout {
func (s *FilesystemState) notify(ctx context.Context, notif *notification) error {
if s.Stdout {
writeToStdout(notif)
}
if len(config.Email) > 0 {
if err := sendEmail(ctx, config.Email, notif); err != nil {
if len(s.Email) > 0 {
if err := sendEmail(ctx, s.Email, notif); err != nil {
return err
}
}
if config.Script != "" {
if err := execScript(ctx, config.Script, notif); err != nil {
if s.Script != "" {
if err := execScript(ctx, s.Script, notif); err != nil {
return err
}
}
if config.ScriptDir != "" {
if err := execScriptDir(ctx, config.ScriptDir, notif); err != nil {
if s.ScriptDir != "" {
if err := execScriptDir(ctx, s.ScriptDir, notif); err != nil {
return err
}
}
@ -57,25 +57,25 @@ func notify(ctx context.Context, config *Config, notif notification) error {
return nil
}
func writeToStdout(notif notification) {
func writeToStdout(notif *notification) {
stdoutMu.Lock()
defer stdoutMu.Unlock()
os.Stdout.WriteString(notif.Text() + "\n")
os.Stdout.WriteString(notif.text + "\n")
}
func sendEmail(ctx context.Context, to []string, notif notification) error {
func sendEmail(ctx context.Context, to []string, notif *notification) error {
stdin := new(bytes.Buffer)
stderr := new(bytes.Buffer)
fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.Summary())
fmt.Fprintf(stdin, "Subject: [certspotter] %s\n", notif.summary)
fmt.Fprintf(stdin, "Date: %s\n", time.Now().Format(mailDateFormat))
fmt.Fprintf(stdin, "Message-ID: <%s>\n", generateMessageID())
fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
fmt.Fprintf(stdin, "\n")
fmt.Fprint(stdin, notif.Text())
fmt.Fprint(stdin, notif.text)
args := []string{"-i", "--"}
args = append(args, to...)
@ -95,12 +95,12 @@ func sendEmail(ctx context.Context, to []string, notif notification) error {
}
}
func execScript(ctx context.Context, scriptName string, notif notification) error {
func execScript(ctx context.Context, scriptName string, notif *notification) error {
stderr := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, scriptName)
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, notif.Environ()...)
cmd.Env = append(cmd.Env, notif.environ...)
cmd.Stderr = stderr
if err := cmd.Run(); err == nil {
@ -116,7 +116,7 @@ func execScript(ctx context.Context, scriptName string, notif notification) erro
}
}
func execScriptDir(ctx context.Context, dirPath string, notif notification) error {
func execScriptDir(ctx context.Context, dirPath string, notif *notification) error {
dirents, err := os.ReadDir(dirPath)
if errors.Is(err, fs.ErrNotExist) {
return nil

78
monitor/process.go
View File

@ -13,19 +13,14 @@ import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"software.sslmate.com/src/certspotter"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/merkletree"
)
type logEntry struct {
type LogEntry struct {
Log *loglist.Log
Index uint64
LeafInput []byte
@ -33,7 +28,7 @@ type logEntry struct {
LeafHash merkletree.Hash
}
func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error {
func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error {
leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
@ -48,7 +43,7 @@ func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error
}
}
func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error {
func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error {
certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
@ -69,7 +64,7 @@ func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, c
return processCertificate(ctx, config, entry, certInfo, chain)
}
func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error {
func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error {
certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
@ -87,7 +82,7 @@ func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry
return processCertificate(ctx, config, entry, certInfo, chain)
}
func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
identifiers, err := certInfo.ParseIdentifiers()
if err != nil {
return processMalformedLogEntry(ctx, config, entry, err)
@ -97,7 +92,7 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce
return nil
}
cert := &discoveredCert{
cert := &DiscoveredCert{
WatchItem: watchItem,
LogEntry: entry,
Info: certInfo,
@ -108,68 +103,15 @@ func processCertificate(ctx context.Context, config *Config, entry *logEntry, ce
Identifiers: identifiers,
}
var notifiedPath string
if config.SaveCerts {
hexFingerprint := hex.EncodeToString(cert.SHA256[:])
prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2])
var (
notifiedFilename = "." + hexFingerprint + ".notified"
certFilename = hexFingerprint + ".pem"
jsonFilename = hexFingerprint + ".v1.json"
textFilename = hexFingerprint + ".txt"
legacyCertFilename = hexFingerprint + ".cert.pem"
legacyPrecertFilename = hexFingerprint + ".precert.pem"
)
for _, filename := range []string{notifiedFilename, legacyCertFilename, legacyPrecertFilename} {
if fileExists(filepath.Join(prefixPath, filename)) {
return nil
}
}
if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.SHA256, err)
}
notifiedPath = filepath.Join(prefixPath, notifiedFilename)
cert.CertPath = filepath.Join(prefixPath, certFilename)
cert.JSONPath = filepath.Join(prefixPath, jsonFilename)
cert.TextPath = filepath.Join(prefixPath, textFilename)
if err := cert.save(); err != nil {
return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
}
} else {
// TODO-4: save cert to temporary files, and defer their unlinking
}
if err := notify(ctx, config, cert); err != nil {
return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.SHA256, err)
}
if notifiedPath != "" {
if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
return fmt.Errorf("error saving certificate %x: %w", cert.SHA256, err)
}
if err := config.State.NotifyCert(ctx, cert); err != nil {
return fmt.Errorf("error notifying about certificate %x: %w", cert.SHA256, err)
}
return nil
}
func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error {
dirPath := filepath.Join(config.StateDir, "logs", entry.Log.LogID.Base64URLString(), "malformed_entries")
malformed := &malformedLogEntry{
Entry: entry,
Error: parseError.Error(),
EntryPath: filepath.Join(dirPath, fmt.Sprintf("%d.json", entry.Index)),
TextPath: filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index)),
}
if err := malformed.save(); err != nil {
return fmt.Errorf("error saving malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
}
if err := notify(ctx, config, malformed); err != nil {
func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error {
if err := config.State.NotifyMalformedEntry(ctx, entry, parseError.Error()); err != nil {
return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
}
return nil

61
monitor/state.go Normal file
View File

@ -0,0 +1,61 @@
// Copyright (C) 2024 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"context"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/merkletree"
"time"
)
type LogState struct {
DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"`
LastSuccess time.Time `json:"last_success"`
}
type StateProvider interface {
// Initialize the state. Called before any other method in this interface.
// Idempotent: returns nil if the state is already initialized.
Prepare(context.Context) error
// Initialize the state for the given log. Called before any other method
// with the log ID. Idempotent: returns nil if log state already initialized.
PrepareLog(context.Context, LogID) error
// Store log state for retrieval by LoadLogState.
StoreLogState(context.Context, LogID, *LogState) error
// Load log state that was previously stored with StoreLogState.
// Returns nil, nil if StoreLogState has not been called yet for this log.
LoadLogState(context.Context, LogID) (*LogState, error)
// Store STH for retrieval by LoadSTHs. If an STH with the same
// timestamp and root hash is already stored, this STH can be ignored.
StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error
// Load all STHs for this log previously stored with StoreSTH.
// The returned slice must be sorted by tree size.
LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error)
// Remove an STH so it is no longer returned by LoadSTHs.
RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error
// Called when a certificate matching the watch list is discovered.
NotifyCert(context.Context, *DiscoveredCert) error
// Called when certspotter fails to parse a log entry.
NotifyMalformedEntry(ctx context.Context, entry *LogEntry, parseError string) error
// Called when a health check fails.
NotifyHealthCheckFailure(context.Context, HealthCheckFailure) error
}

4
monitor/statedir.go
View File

@ -76,13 +76,13 @@ func migrateLogStateDirV1(dir string) error {
return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
}
stateFile := stateFile{
stateFile := LogState{
DownloadPosition: &tree,
VerifiedPosition: &tree,
VerifiedSTH: &sth,
LastSuccess: time.Now().UTC(),
}
if stateFile.store(filepath.Join(dir, "state.json")); err != nil {
if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil {
return err
}

42
monitor/statefile.go
View File

@ -1,42 +0,0 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"encoding/json"
"fmt"
"os"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/merkletree"
"time"
)
type stateFile struct {
DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"`
LastSuccess time.Time `json:"last_success"`
}
func loadStateFile(filePath string) (*stateFile, error) {
fileBytes, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
file := new(stateFile)
if err := json.Unmarshal(fileBytes, file); err != nil {
return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
}
return file, nil
}
func (file *stateFile) store(filePath string) error {
return writeJSONFile(filePath, file, 0666)
}