diff --git a/cmd/certspotter/main.go b/cmd/certspotter/main.go index da1e4ed..d099a5d 100644 --- a/cmd/certspotter/main.go +++ b/cmd/certspotter/main.go @@ -1,4 +1,4 @@ -// Copyright (C) 2016 Opsmate, Inc. +// Copyright (C) 2016, 2023 Opsmate, Inc. // // This Source Code Form is subject to the terms of the Mozilla // Public License, v. 2.0. If a copy of the MPL was not distributed @@ -10,224 +10,169 @@ package main import ( - "bufio" + "context" + "errors" "flag" "fmt" - "io" + "io/fs" + insecurerand "math/rand" "os" + "os/signal" "path/filepath" + "runtime/debug" "strings" + "syscall" "time" - "golang.org/x/net/idna" - - "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/cmd" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/monitor" ) +var programName = os.Args[0] + +const defaultLogList = "https://loglist.certspotter.org/monitor.json" + +func certspotterVersion() string { + info, ok := debug.ReadBuildInfo() + if !ok { + return "unknown" + } + if strings.HasPrefix(info.Main.Version, "v") { + return info.Main.Version + } + var vcs, vcsRevision, vcsModified string + for _, s := range info.Settings { + switch s.Key { + case "vcs": + vcs = s.Value + case "vcs.revision": + vcsRevision = s.Value + case "vcs.modified": + vcsModified = s.Value + } + } + if vcs == "git" && vcsRevision != "" && vcsModified == "true" { + return vcsRevision + "+" + } else if vcs == "git" && vcsRevision != "" { + return vcsRevision + } + return "unknown" +} + +func homedir() string { + homedir, err := os.UserHomeDir() + if err != nil { + panic(fmt.Errorf("unable to determine home directory: %w", err)) + } + return homedir +} func defaultStateDir() string { if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" { return envVar } else { - return cmd.DefaultStateDir("certspotter") + return filepath.Join(homedir(), ".certspotter") } } func defaultConfigDir() string { if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" { return envVar } else { - return cmd.DefaultConfigDir("certspotter") + return filepath.Join(homedir(), ".certspotter") } } -func trimTrailingDots(value string) string { - length := len(value) - for length > 0 && value[length-1] == '.' { - length-- - } - return value[0:length] -} - -var stateDir = flag.String("state_dir", defaultStateDir(), "Directory for storing state") -var watchlistFilename = flag.String("watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing identifiers to watch (- for stdin)") - -type watchlistItem struct { - Domain []string - AcceptSuffix bool - ValidAt *time.Time // optional -} - -var watchlist []watchlistItem - -func parseWatchlistItem(str string) (watchlistItem, error) { - fields := strings.Fields(str) - if len(fields) == 0 { - return watchlistItem{}, fmt.Errorf("Empty domain") - } - domain := fields[0] - var validAt *time.Time = nil - - // parse options - for i := 1; i < len(fields); i++ { - chunks := strings.SplitN(fields[i], ":", 2) - if len(chunks) != 2 { - return watchlistItem{}, fmt.Errorf("Missing Value `%s'", fields[i]) - } - switch chunks[0] { - case "valid_at": - validAtTime, err := time.Parse("2006-01-02", chunks[1]) - if err != nil { - return watchlistItem{}, fmt.Errorf("Invalid Date `%s': %s", chunks[1], err) - } - validAt = &validAtTime - default: - return watchlistItem{}, fmt.Errorf("Unknown Option `%s'", fields[i]) - } - } - - // parse domain - // "." as in root zone (matches everything) - if domain == "." { - return watchlistItem{ - Domain: []string{}, - AcceptSuffix: true, - ValidAt: validAt, - }, nil - } - - acceptSuffix := false - if strings.HasPrefix(domain, ".") { - acceptSuffix = true - domain = domain[1:] - } - - asciiDomain, err := idna.ToASCII(strings.ToLower(trimTrailingDots(domain))) +func readWatchListFile(filename string) (monitor.WatchList, error) { + file, err := os.Open(filename) if err != nil { - return watchlistItem{}, fmt.Errorf("Invalid domain `%s': %s", domain, err) - } - return watchlistItem{ - Domain: strings.Split(asciiDomain, "."), - AcceptSuffix: acceptSuffix, - ValidAt: validAt, - }, nil -} - -func readWatchlist(reader io.Reader) ([]watchlistItem, error) { - items := []watchlistItem{} - scanner := bufio.NewScanner(reader) - for scanner.Scan() { - line := scanner.Text() - if line == "" || strings.HasPrefix(line, "#") { - continue + var pathErr *fs.PathError + if errors.As(err, &pathErr) { + err = pathErr.Err } - item, err := parseWatchlistItem(line) - if err != nil { - return nil, err - } - items = append(items, item) + return nil, err } - return items, scanner.Err() + defer file.Close() + return monitor.ReadWatchList(file) } -func dnsLabelMatches(certLabel string, watchLabel string) bool { - // For fail-safe behavior, if a label was unparsable, it matches everything. - // Similarly, redacted labels match everything, since the label _might_ be - // for a name we're interested in. - - return certLabel == "*" || - certLabel == "?" || - certLabel == certspotter.UnparsableDNSLabelPlaceholder || - certspotter.MatchesWildcard(watchLabel, certLabel) -} - -func dnsNameMatches(dnsName []string, watchDomain []string, acceptSuffix bool) bool { - for len(dnsName) > 0 && len(watchDomain) > 0 { - certLabel := dnsName[len(dnsName)-1] - watchLabel := watchDomain[len(watchDomain)-1] - - if !dnsLabelMatches(certLabel, watchLabel) { - return false - } - - dnsName = dnsName[:len(dnsName)-1] - watchDomain = watchDomain[:len(watchDomain)-1] - } - - return len(watchDomain) == 0 && (acceptSuffix || len(dnsName) == 0) -} - -func anyDnsNameIsWatched(info *certspotter.EntryInfo) bool { - dnsNames := info.Identifiers.DNSNames - matched := false - for _, dnsName := range dnsNames { - labels := strings.Split(dnsName, ".") - for _, item := range watchlist { - if dnsNameMatches(labels, item.Domain, item.AcceptSuffix) { - if item.ValidAt != nil { - // BygoneSSL Check - // was the SSL certificate issued before the domain was registered - // and valid after - if item.ValidAt.Before(*info.CertInfo.NotAfter()) && - item.ValidAt.After(*info.CertInfo.NotBefore()) { - info.Bygone = true - return true - } - } - // keep iterating in case another domain watched matches valid_at - matched = true - } - } - } - return matched -} - -func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) { - info := certspotter.EntryInfo{ - LogUri: scanner.LogUri, - Entry: entry, - IsPrecert: certspotter.IsPrecert(entry), - FullChain: certspotter.GetFullChain(entry), - } - - info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry) - - if info.CertInfo != nil { - info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers() - } - - // Fail safe behavior: if info.Identifiers is nil (which is caused by a - // parse error), report the certificate because we can't say for sure it - // doesn't match a domain we care about. We try very hard to make sure - // parsing identifiers always succeeds, so false alarms should be rare. - if info.Identifiers == nil || anyDnsNameIsWatched(&info) { - cmd.LogEntry(&info) +func appendFunc(slice *[]string) func(string) error { + return func(value string) error { + *slice = append(*slice, value) + return nil } } func main() { - cmd.ParseFlags() + insecurerand.Seed(time.Now().UnixNano()) // TODO: remove after upgrading to Go 1.20 - if *watchlistFilename == "-" { - var err error - watchlist, err = readWatchlist(os.Stdin) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: (stdin): %s\n", os.Args[0], err) - os.Exit(1) - } - } else { - file, err := os.Open(*watchlistFilename) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err) - os.Exit(1) - } - watchlist, err = readWatchlist(file) - file.Close() - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err) - os.Exit(1) - } + // TODO-3: set loglist.UserAgent + + var flags struct { + batchSize int // TODO-4: respect this option + email []string + logs string + noSave bool + script string + startAtEnd bool + stateDir string + stdout bool + verbose bool + version bool + watchlist string + } + flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)") + flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email)) + flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor") + flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory") + flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered") + flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)") + flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates") + flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout") + flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose") + flag.BoolVar(&flags.version, "version", false, "Print version and exit") + flag.StringVar(&flags.watchlist, "watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing domain names to watch") + flag.Parse() + + if flags.version { + fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion()) + os.Exit(0) } - os.Exit(cmd.Main(*stateDir, processEntry)) + if len(flags.email) == 0 && len(flags.script) == 0 && flags.stdout == false { + fmt.Fprintf(os.Stderr, "%s: at least one of -email, -script, or -stdout must be specified (see -help for details)\n", programName) + os.Exit(2) + } + + config := &monitor.Config{ + LogListSource: flags.logs, + StateDir: flags.stateDir, + SaveCerts: !flags.noSave, + StartAtEnd: flags.startAtEnd, + Verbose: flags.verbose, + Script: flags.script, + Email: flags.email, + Stdout: flags.stdout, + } + + if flags.watchlist == "-" { + watchlist, err := monitor.ReadWatchList(os.Stdin) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err) + os.Exit(1) + } + config.WatchList = watchlist + } else { + watchlist, err := readWatchListFile(flags.watchlist) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err) + os.Exit(1) + } + config.WatchList = watchlist + } + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) { + fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err) + os.Exit(1) + } } diff --git a/go.mod b/go.mod index f56df2d..53397d0 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,10 @@ module software.sslmate.com/src/certspotter -go 1.17 +go 1.19 require ( - golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/exp v0.0.0-20230118134722-a68e582fa157 // indirect + golang.org/x/net v0.5.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/text v0.6.0 // indirect ) diff --git a/go.sum b/go.sum index ce35e63..51a8c19 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,12 @@ +golang.org/x/exp v0.0.0-20230118134722-a68e582fa157 h1:fiNkyhJPUvxbRPbCqY/D9qdjmPzfHcpK3P4bM4gioSY= +golang.org/x/exp v0.0.0-20230118134722-a68e582fa157/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= diff --git a/monitor/config.go b/monitor/config.go new file mode 100644 index 0000000..46eb100 --- /dev/null +++ b/monitor/config.go @@ -0,0 +1,22 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +type Config struct { + LogListSource string + StateDir string + StartAtEnd bool + WatchList WatchList + Verbose bool + SaveCerts bool + Script string + Email []string + Stdout bool +} diff --git a/monitor/daemon.go b/monitor/daemon.go new file mode 100644 index 0000000..71b6196 --- /dev/null +++ b/monitor/daemon.go @@ -0,0 +1,142 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "errors" + "fmt" + "golang.org/x/sync/errgroup" + "log" + insecurerand "math/rand" + "software.sslmate.com/src/certspotter/loglist" + "time" +) + +const ( + reloadLogListIntervalMin = 3 * time.Second // TODO-3: use 30 * time.Minute + reloadLogListIntervalMax = 9 * time.Second // TODO-3: use 90 * time.Minute + healthCheckInterval = 24 * time.Hour +) + +func randomDuration(min, max time.Duration) time.Duration { + return min + time.Duration(insecurerand.Int63n(int64(max-min+1))) +} + +func reloadLogListInterval() time.Duration { + return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax) +} + +type task struct { + stop context.CancelFunc +} + +type daemon struct { + config *Config + taskgroup *errgroup.Group + tasks map[LogID]task + logsLoadedAt time.Time +} + +func (daemon *daemon) healthCheck(ctx context.Context) error { // TODO-2 + return nil +} + +func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task { + ctx, cancel := context.WithCancel(ctx) + daemon.taskgroup.Go(func() error { + defer cancel() + err := monitorLogContinously(ctx, daemon.config, ctlog) + if daemon.config.Verbose { + log.Printf("task for log %s stopped with error %s", ctlog.URL, err) + } + if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) { + return nil + } else { + return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err) + } + }) + return task{stop: cancel} +} + +func (daemon *daemon) loadLogList(ctx context.Context) error { + loglist, err := getLogList(ctx, daemon.config.LogListSource) + if err != nil { + return err + } + if daemon.config.Verbose { + log.Printf("fetched %d logs from %q", len(loglist), daemon.config.LogListSource) + } + + for logID, task := range daemon.tasks { + if _, exists := loglist[logID]; exists { + continue + } + if daemon.config.Verbose { + log.Printf("stopping task for log %s", logID.Base64String()) + } + task.stop() + delete(daemon.tasks, logID) + } + for logID, ctlog := range loglist { + if _, isRunning := daemon.tasks[logID]; isRunning { + continue + } + if daemon.config.Verbose { + log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL) + } + daemon.tasks[logID] = daemon.startTask(ctx, ctlog) + } + daemon.logsLoadedAt = time.Now() + return nil +} + +func (daemon *daemon) run(ctx context.Context) error { + if err := prepareStateDir(daemon.config.StateDir); err != nil { + return fmt.Errorf("error preparing state directory: %w", err) + } + + if err := daemon.loadLogList(ctx); err != nil { + return fmt.Errorf("error loading log list: %w", err) + } + + reloadLogListTicker := time.NewTicker(reloadLogListInterval()) + defer reloadLogListTicker.Stop() + + healthCheckTicker := time.NewTicker(healthCheckInterval) + defer healthCheckTicker.Stop() + + for ctx.Err() == nil { + select { + case <-ctx.Done(): + case <-reloadLogListTicker.C: + if err := daemon.loadLogList(ctx); err != nil { + recordError(fmt.Errorf("error reloading log list (will try again later): %w", err)) + } + reloadLogListTicker.Reset(reloadLogListInterval()) + case <-healthCheckTicker.C: + if err := daemon.healthCheck(ctx); err != nil { + return err + } + } + } + return ctx.Err() +} + +func Run(ctx context.Context, config *Config) error { + group, ctx := errgroup.WithContext(ctx) + daemon := &daemon{ + config: config, + taskgroup: group, + tasks: make(map[LogID]task), + } + group.Go(func() error { return daemon.run(ctx) }) + return group.Wait() +} diff --git a/monitor/discoveredcert.go b/monitor/discoveredcert.go new file mode 100644 index 0000000..a644f01 --- /dev/null +++ b/monitor/discoveredcert.go @@ -0,0 +1,188 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "strings" + "time" + + "software.sslmate.com/src/certspotter" + "software.sslmate.com/src/certspotter/ct" +) + +type discoveredCert struct { + WatchItem WatchItem + LogEntry *logEntry + Info *certspotter.CertInfo + Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate + LeafSHA256 [32]byte // computed over Chain[0] + Identifiers *certspotter.Identifiers + CertPath string // empty if not saved on the filesystem + JSONPath string // empty if not saved on the filesystem + TextPath string // empty if not saved on the filesystem +} + +func (cert *discoveredCert) pemChain() []byte { + var buffer bytes.Buffer + for _, certBytes := range cert.Chain { + if err := pem.Encode(&buffer, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }); err != nil { + panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err)) + } + } + return buffer.Bytes() +} + +func (cert *discoveredCert) json() []byte { + pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey()) + + object := map[string]any{ + "cert_sha256": hex.EncodeToString(cert.LeafSHA256[:]), + "pubkey_sha256": hex.EncodeToString(pubkeySha256[:]), + "issuer_der": cert.Info.TBS.Issuer.FullBytes, + "subject_der": cert.Info.TBS.Subject.FullBytes, + "dns_names": cert.Identifiers.DNSNames, + "ip_addresses": cert.Identifiers.IPAddrs, + } + + if cert.Info.ValidityParseError == nil { + object["not_before"] = cert.Info.Validity.NotBefore + object["not_after"] = cert.Info.Validity.NotAfter + } else { + object["not_before"] = nil + object["not_after"] = nil + } + + if cert.Info.SerialNumberParseError == nil { + object["serial_number"] = fmt.Sprintf("%x", cert.Info.SerialNumber) + } else { + object["serial_number"] = nil + } + + jsonBytes, err := json.Marshal(object) + if err != nil { + panic(fmt.Errorf("encoding certificate as JSON failed unexpectedly: %w", err)) + } + return jsonBytes +} + +func (cert *discoveredCert) save() error { + if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil { + return err + } + if err := writeFile(cert.JSONPath, cert.json(), 0666); err != nil { + return err + } + if err := writeFile(cert.TextPath, []byte(cert.Text()), 0666); err != nil { + return err + } + return nil +} + +func (cert *discoveredCert) Environ() []string { + pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey()) + + env := []string{ + "EVENT=discovered_cert", + "SUMMARY=certificate discovered for " + cert.WatchItem.String(), + "CERT_PARSEABLE=yes", // backwards compat; not documented (TODO-3: consider removing) + "LOG_URI=" + cert.LogEntry.Log.URL, + "ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index), + "WATCH_ITEM=" + cert.WatchItem.String(), + "CERT_SHA256=" + hex.EncodeToString(cert.LeafSHA256[:]), + "FINGERPRINT=" + hex.EncodeToString(cert.LeafSHA256[:]), // backwards compat; not documented (TODO-3: consider removing) + "PUBKEY_SHA256=" + hex.EncodeToString(pubkeySha256[:]), + "PUBKEY_HASH=" + hex.EncodeToString(pubkeySha256[:]), // backwards compat; not documented (TODO-3: consider removing) + "CERT_FILENAME=" + cert.CertPath, + "JSON_FILENAME=" + cert.JSONPath, + "TEXT_FILENAME=" + cert.TextPath, + } + + if cert.Info.ValidityParseError == nil { + env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String()) + env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix())) + env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339)) + env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String()) + env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix())) + env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339)) + } else { + env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error()) + } + + if cert.Info.SubjectParseError == nil { + env = append(env, "SUBJECT_DN="+cert.Info.Subject.String()) + } else { + env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error()) + } + + if cert.Info.IssuerParseError == nil { + env = append(env, "ISSUER_DN="+cert.Info.Issuer.String()) + } else { + env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error()) + } + + // TODO-3: consider removing $SERIAL due to misuse potential, or renaming to $SERIAL_NUMBER + if cert.Info.SerialNumberParseError == nil { + env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber)) + } else { + env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error()) + } + + return env +} + +func (cert *discoveredCert) Text() string { + // TODO-3: improve the output: include WatchItem, indicate hash algorithm. look at sslmate email for inspiration + + text := new(strings.Builder) + writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } + + pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey()) + + fmt.Fprintf(text, "%x:\n", cert.LeafSHA256) + for _, dnsName := range cert.Identifiers.DNSNames { + writeField("DNS Name", dnsName) + } + for _, ipaddr := range cert.Identifiers.IPAddrs { + writeField("IP Address", ipaddr) + } + writeField("Pubkey", hex.EncodeToString(pubkeySha256[:])) + if cert.Info.IssuerParseError == nil { + writeField("Issuer", cert.Info.Issuer) + } else { + writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError)) + } + if cert.Info.ValidityParseError == nil { + writeField("Not Before", cert.Info.Validity.NotBefore) + writeField("Not After", cert.Info.Validity.NotAfter) + } else { + writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) + writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) + } + writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL)) // TODO-3: include entry type here? + writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.LeafSHA256[:])) + if cert.CertPath != "" { + writeField("Filename", cert.CertPath) + } + + return text.String() +} + +func (cert *discoveredCert) EmailSubject() string { + return fmt.Sprintf("[certspotter] Certificate Discovered for %s", cert.WatchItem) +} diff --git a/monitor/errors.go b/monitor/errors.go new file mode 100644 index 0000000..f43df1a --- /dev/null +++ b/monitor/errors.go @@ -0,0 +1,18 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "log" +) + +func recordError(err error) { + log.Print(err) +} diff --git a/monitor/fileutils.go b/monitor/fileutils.go new file mode 100644 index 0000000..029172c --- /dev/null +++ b/monitor/fileutils.go @@ -0,0 +1,42 @@ +// Copyright (C) 2017, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "os" +) + +func randomFileSuffix() string { + var randomBytes [12]byte + if _, err := rand.Read(randomBytes[:]); err != nil { + panic(err) + } + return hex.EncodeToString(randomBytes[:]) +} + +func writeFile(filename string, data []byte, perm os.FileMode) error { + tempname := filename + ".tmp." + randomFileSuffix() + if err := os.WriteFile(tempname, data, perm); err != nil { + return fmt.Errorf("error writing %s: %w", filename, err) + } + if err := os.Rename(tempname, filename); err != nil { + os.Remove(tempname) + return fmt.Errorf("error writing %s: %w", filename, err) + } + return nil +} + +func fileExists(filename string) bool { + _, err := os.Lstat(filename) + return err == nil +} diff --git a/monitor/healthcheck.go b/monitor/healthcheck.go new file mode 100644 index 0000000..9e5e79e --- /dev/null +++ b/monitor/healthcheck.go @@ -0,0 +1,102 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "fmt" + "strings" + "time" + + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" +) + +type staleSTHEvent struct { + Log *loglist.Log + LastSuccess time.Time + LatestSTH *ct.SignedTreeHead // may be nil +} +type backlogEvent struct { + Log *loglist.Log + LatestSTH *ct.SignedTreeHead + Backlog uint64 + Position uint64 +} +type staleLogListEvent struct { + Source string + LastSuccess time.Time + LastError string + LastErrorTime time.Time +} + +func (e *staleSTHEvent) Environ() []string { + return []string{ + "EVENT=error", + "SUMMARY=" + fmt.Sprintf("unable to contact %s since %s", e.Log.URL, e.LastSuccess), + } +} +func (e *backlogEvent) Environ() []string { + return []string{ + "EVENT=error", + "SUMMARY=" + fmt.Sprintf("backlog of size %d from %s", e.Backlog, e.Log.URL), + } +} +func (e *staleLogListEvent) Environ() []string { + return []string{ + "EVENT=error", + "SUMMARY=" + fmt.Sprintf("unable to retrieve log list since %s: %s", e.LastSuccess, e.LastError), + } +} + +func (e *staleSTHEvent) EmailSubject() string { + return fmt.Sprintf("[certspotter] Unable to contact %s since %s", e.Log.URL, e.LastSuccess) +} +func (e *backlogEvent) EmailSubject() string { + return fmt.Sprintf("[certspotter] Backlog of size %d from %s", e.Backlog, e.Log.URL) +} +func (e *staleLogListEvent) EmailSubject() string { + return fmt.Sprintf("[certspotter] Unable to retrieve log list since %s", e.LastSuccess) +} + +func (e *staleSTHEvent) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "For details, see certspotter's stderr output.\n") + fmt.Fprintf(text, "\n") + if e.LatestSTH != nil { + fmt.Fprintf(text, "Latest known log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.Timestamp) + } else { + fmt.Fprintf(text, "Latest known log size = none\n") + } + return text.String() +} +func (e *backlogEvent) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n") + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.Timestamp) + fmt.Fprintf(text, "Current position = %d\n", e.Position) + fmt.Fprintf(text, " Backlog = %d\n", e.Backlog) + return text.String() +} +func (e *staleLogListEvent) Text() string { + text := new(strings.Builder) + fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError) + fmt.Fprintf(text, "\n") + fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n") + return text.String() +} + +// TODO-3: make the errors more actionable diff --git a/monitor/loglist.go b/monitor/loglist.go new file mode 100644 index 0000000..122231f --- /dev/null +++ b/monitor/loglist.go @@ -0,0 +1,40 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "fmt" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" +) + +type LogID = ct.SHA256Hash + +func getLogList(ctx context.Context, source string) (map[LogID]*loglist.Log, error) { + // TODO-4: pass context to loglist.Load + // TODO-3: If-Modified-Since / If-None-Match support + list, err := loglist.Load(source) + if err != nil { + return nil, err + } + + logs := make(map[LogID]*loglist.Log) + for operatorIndex := range list.Operators { + for logIndex := range list.Operators[operatorIndex].Logs { + log := &list.Operators[operatorIndex].Logs[logIndex] + if _, exists := logs[log.LogID]; exists { + return nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String()) + } + logs[log.LogID] = log + } + } + return logs, nil +} diff --git a/monitor/malformed.go b/monitor/malformed.go new file mode 100644 index 0000000..310d999 --- /dev/null +++ b/monitor/malformed.go @@ -0,0 +1,48 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "fmt" + "strings" +) + +type malformedLogEntry struct { + Entry *logEntry + Error string +} + +func (malformed *malformedLogEntry) Environ() []string { + return []string{ + "EVENT=discovered_cert", + "SUMMARY=" + fmt.Sprintf("unable to parse entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL), + "LOG_URI=" + malformed.Entry.Log.URL, + "ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index), + "LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(), + "PARSE_ERROR=" + malformed.Error, + "CERT_PARSEABLE=no", // backwards compat; not documented (TODO-3: consider removing) + } +} + +func (malformed *malformedLogEntry) Text() string { + text := new(strings.Builder) + writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } + + fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n") + writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL)) + writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String()) + writeField("Error", malformed.Error) + + return text.String() +} + +func (malformed *malformedLogEntry) EmailSubject() string { + return fmt.Sprintf("[certspotter] Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL) +} diff --git a/monitor/monitor.go b/monitor/monitor.go new file mode 100644 index 0000000..1f403ab --- /dev/null +++ b/monitor/monitor.go @@ -0,0 +1,294 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + "time" + + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/ct/client" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" +) + +const ( + maxGetEntriesSize = 1000 + //monitorLogInterval = 5 * time.Minute // TODO-3 + monitorLogInterval = 15 * time.Second +) + +func isFatalLogError(err error) bool { + return errors.Is(err, context.Canceled) +} + +func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) { + logKey, err := x509.ParsePKIXPublicKey(ctlog.Key) + if err != nil { + return nil, fmt.Errorf("error parsing log key: %w", err) + } + verifier, err := ct.NewSignatureVerifier(logKey) + if err != nil { + return nil, fmt.Errorf("error with log key: %w", err) + } + return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil +} + +func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error { + logClient, err := newLogClient(ctlog) + if err != nil { + return err + } + + ticker := time.NewTicker(monitorLogInterval) + defer ticker.Stop() + + for ctx.Err() == nil { + if err := monitorLog(ctx, config, ctlog, logClient); err != nil { + return err + } + select { + case <-ctx.Done(): + case <-ticker.C: + } + } + return ctx.Err() +} + +func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var ( + stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString()) + stateFilePath = filepath.Join(stateDirPath, "state.json") + sthsDirPath = filepath.Join(stateDirPath, "unverified_sths") + ) + for _, dirPath := range []string{stateDirPath, sthsDirPath} { + if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("error creating state directory: %w", err) + } + } + + startTime := time.Now() + latestSTH, err := logClient.GetSTH(ctx) + if isFatalLogError(err) { + return err + } else if err != nil { + recordError(fmt.Errorf("error fetching latest STH for %s: %w", ctlog.URL, err)) + return nil + } + latestSTH.LogID = ctlog.LogID + if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil { + return fmt.Errorf("error storing latest STH: %w", err) + } + + state, err := loadStateFile(stateFilePath) + if errors.Is(err, fs.ErrNotExist) { + if config.StartAtEnd { + tree, err := reconstructTree(ctx, logClient, latestSTH) + if isFatalLogError(err) { + return err + } else if err != nil { + recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err)) + return nil + } + state = &stateFile{ + DownloadPosition: tree, + VerifiedPosition: tree, + VerifiedSTH: latestSTH, + LastSuccess: startTime.UTC(), + } + } else { + state = &stateFile{ + DownloadPosition: merkletree.EmptyCollapsedTree(), + VerifiedPosition: merkletree.EmptyCollapsedTree(), + VerifiedSTH: nil, + LastSuccess: startTime.UTC(), + } + } + if config.Verbose { + log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size()) + } + if err := state.store(stateFilePath); err != nil { + return fmt.Errorf("error storing state file: %w", err) + } + } else if err != nil { + return fmt.Errorf("error loading state file: %w", err) + } + + sths, err := loadSTHsFromDir(sthsDirPath) + if err != nil { + return fmt.Errorf("error loading STHs directory: %w", err) + } + + for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() { + // TODO-4: audit sths[0] against state.VerifiedSTH + if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil { + return fmt.Errorf("error removing STH: %w", err) + } + sths = sths[1:] + } + + defer func() { + if config.Verbose { + log.Printf("saving state in defer for %s", ctlog.URL) + } + if err := state.store(stateFilePath); err != nil && returnedErr == nil { + returnedErr = fmt.Errorf("error storing state file: %w", err) + } + }() + + if len(sths) == 0 { + state.LastSuccess = startTime.UTC() + return nil + } + + var ( + downloadBegin = state.DownloadPosition.Size() + downloadEnd = sths[len(sths)-1].TreeSize + entries = make(chan client.GetEntriesItem, maxGetEntriesSize) + downloadErr error + ) + if config.Verbose { + log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd) + } + go func() { + defer close(entries) + downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd) + }() + for rawEntry := range entries { + entry := &logEntry{ + Log: ctlog, + Index: state.DownloadPosition.Size(), + LeafInput: rawEntry.LeafInput, + ExtraData: rawEntry.ExtraData, + LeafHash: merkletree.HashLeaf(rawEntry.LeafInput), + } + if err := processLogEntry(ctx, config, entry); err != nil { + return fmt.Errorf("error processing entry %d: %w", entry.Index, err) + } + + state.DownloadPosition.Add(entry.LeafHash) + rootHash := state.DownloadPosition.CalculateRoot() + shouldSaveState := state.DownloadPosition.Size()%10000 == 0 + + for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize { + if merkletree.Hash(sths[0].SHA256RootHash) != rootHash { + recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash)) + + state.DownloadPosition = state.VerifiedPosition + if err := state.store(stateFilePath); err != nil { + return fmt.Errorf("error storing state file: %w", err) + } + return nil + } + + state.VerifiedPosition = state.DownloadPosition + state.VerifiedSTH = sths[0] + shouldSaveState = true + if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil { + return fmt.Errorf("error removing verified STH: %w", err) + } + + sths = sths[1:] + } + + if shouldSaveState { + if err := state.store(stateFilePath); err != nil { + return fmt.Errorf("error storing state file: %w", err) + } + } + } + + if isFatalLogError(downloadErr) { + return downloadErr + } else if downloadErr != nil { + recordError(fmt.Errorf("error downloading entries from %s: %w", ctlog.URL, downloadErr)) + return nil + } + + if config.Verbose { + log.Printf("finished downloading entries from %s", ctlog.URL) + } + + state.LastSuccess = startTime.UTC() + return nil +} + +func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error { + for begin < end && ctx.Err() == nil { + size := begin - end + if size > maxGetEntriesSize { + size = maxGetEntriesSize + } + entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1) + if err != nil { + return err + } + for _, entry := range entries { + if ctx.Err() != nil { + return ctx.Err() + } + select { + case <-ctx.Done(): + return ctx.Err() + case entriesChan <- entry: + } + } + begin += uint64(len(entries)) + } + return ctx.Err() +} + +func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) { + if sth.TreeSize == 0 { + return merkletree.EmptyCollapsedTree(), nil + } + entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1) + if err != nil { + return nil, err + } + leafHash := merkletree.HashLeaf(entries[0].LeafInput) + + var tree *merkletree.CollapsedTree + if sth.TreeSize > 1 { + auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize) + if err != nil { + return nil, err + } + hashes := make([]merkletree.Hash, len(auditPath)) + for i := range hashes { + copy(hashes[i][:], auditPath[len(auditPath)-i-1]) + } + tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1) + if err != nil { + return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err) + } + } else { + tree = merkletree.EmptyCollapsedTree() + } + + tree.Add(leafHash) + rootHash := tree.CalculateRoot() + if rootHash != merkletree.Hash(sth.SHA256RootHash) { + return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize) + } + + return tree, nil +} diff --git a/monitor/notify.go b/monitor/notify.go new file mode 100644 index 0000000..025b0a5 --- /dev/null +++ b/monitor/notify.go @@ -0,0 +1,151 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" +) + +var stdoutMu sync.Mutex + +type notification interface { + Environ() []string + EmailSubject() string + Text() string +} + +func notify(ctx context.Context, config *Config, notif notification) error { + if config.Stdout { + writeToStdout(notif) + } + + if len(config.Email) > 0 { + if err := sendEmail(ctx, config.Email, notif); err != nil { + return err + } + } + + if config.Script != "" { + if err := execScript(ctx, config.Script, notif); err != nil { + return err + } + } + + return nil +} + +func writeToStdout(notif notification) { + stdoutMu.Lock() + defer stdoutMu.Unlock() + os.Stdout.WriteString(notif.Text() + "\n") +} + +func sendEmail(ctx context.Context, to []string, notif notification) error { + stdin := new(bytes.Buffer) + stderr := new(bytes.Buffer) + + fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", ")) + fmt.Fprintf(stdin, "Subject: %s\n", notif.EmailSubject()) + fmt.Fprintf(stdin, "Mime-Version: 1.0\n") + fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n") + fmt.Fprintf(stdin, "X-Mailer: certspotter\n") + fmt.Fprintf(stdin, "\n") + fmt.Fprint(stdin, notif.Text()) + + args := []string{"-i", "--"} + args = append(args, to...) + + sendmail := exec.CommandContext(ctx, "/usr/sbin/sendmail", args...) + sendmail.Stdin = stdin + sendmail.Stderr = stderr + + if err := sendmail.Run(); err == nil { + return nil + } else if ctx.Err() != nil { + return ctx.Err() + } else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() { + return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String())) + } else { + return fmt.Errorf("error sending email to %v: %w", to, err) + } +} + +func execScript(ctx context.Context, scriptPath string, notif notification) error { + // TODO-3: consider removing directory support (for now), and supporting $PATH lookups + info, err := os.Stat(scriptPath) + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("script %q does not exist", scriptPath) + } else if err != nil { + return fmt.Errorf("error executing script %q: %w", scriptPath, err) + } else if info.IsDir() { + return execScriptDir(ctx, scriptPath, notif) + } else { + return execScriptFile(ctx, scriptPath, notif) + } + +} + +func execScriptDir(ctx context.Context, dirPath string, notif notification) error { + dirents, err := os.ReadDir(dirPath) + if err != nil { + return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err) + } + for _, dirent := range dirents { + if strings.HasPrefix(dirent.Name(), ".") { + continue + } + scriptPath := filepath.Join(dirPath, dirent.Name()) + info, err := os.Stat(scriptPath) + if errors.Is(err, fs.ErrNotExist) { + continue + } else if err != nil { + return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err) + } else if info.Mode().IsRegular() && isExecutable(info.Mode()) { + if err := execScriptFile(ctx, scriptPath, notif); err != nil { + return err + } + } + } + return nil +} + +func execScriptFile(ctx context.Context, scriptPath string, notif notification) error { + stderr := new(bytes.Buffer) + + cmd := exec.CommandContext(ctx, scriptPath) + cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, notif.Environ()...) + cmd.Stderr = stderr + + if err := cmd.Run(); err == nil { + return nil + } else if ctx.Err() != nil { + return ctx.Err() + } else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() { + return fmt.Errorf("script %q exited with code %d and error %q", scriptPath, exitErr.ExitCode(), strings.TrimSpace(stderr.String())) + } else if isExitError { + return fmt.Errorf("script %q terminated by signal with error %q", scriptPath, strings.TrimSpace(stderr.String())) + } else { + return fmt.Errorf("error executing script %q: %w", scriptPath, err) + } +} + +func isExecutable(mode os.FileMode) bool { + return mode&0111 != 0 +} diff --git a/monitor/process.go b/monitor/process.go new file mode 100644 index 0000000..e5fff74 --- /dev/null +++ b/monitor/process.go @@ -0,0 +1,150 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "software.sslmate.com/src/certspotter" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" +) + +type logEntry struct { + Log *loglist.Log + Index uint64 + LeafInput []byte + ExtraData []byte + LeafHash merkletree.Hash +} + +func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error { + leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput)) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err)) + } + switch leaf.TimestampedEntry.EntryType { + case ct.X509LogEntryType: + return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry) + case ct.PrecertLogEntryType: + return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry) + default: + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType)) + } +} + +func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error { + certInfo, err := certspotter.MakeCertInfoFromRawCert(cert) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err)) + } + + chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err)) + } + chain = append([]ct.ASN1Cert{cert}, chain...) + + return processCertificate(ctx, config, entry, certInfo, chain) +} + +func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error { + certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err)) + } + + chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData) + if err != nil { + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err)) + } + + return processCertificate(ctx, config, entry, certInfo, chain) +} + +func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error { + identifiers, err := certInfo.ParseIdentifiers() + if err != nil { + return processMalformedLogEntry(ctx, config, entry, err) + } + matched, watchItem := config.WatchList.Matches(identifiers) + if !matched { + return nil + } + + cert := &discoveredCert{ + WatchItem: watchItem, + LogEntry: entry, + Info: certInfo, + Chain: chain, + LeafSHA256: sha256.Sum256(chain[0]), + Identifiers: identifiers, + } + + var notifiedPath string + if config.SaveCerts { + hexFingerprint := hex.EncodeToString(cert.LeafSHA256[:]) + prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2]) + + for _, suffix := range []string{".notified", ".cert.pem", ".precert.pem"} { + if fileExists(filepath.Join(prefixPath, hexFingerprint+suffix)) { + return nil + } + } + + if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.LeafSHA256, err) + } + + notifiedPath = filepath.Join(prefixPath, hexFingerprint+".notified") // TODO-3: maybe this should be a hidden file? + cert.CertPath = filepath.Join(prefixPath, hexFingerprint+".pem") + cert.JSONPath = filepath.Join(prefixPath, hexFingerprint+".json") // TODO-3: consider using .v1.json extension in case I change the format later? + cert.TextPath = filepath.Join(prefixPath, hexFingerprint+".txt") + + if err := cert.save(); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.LeafSHA256, err) + } + } else { + // TODO-4: save cert to temporary files, and defer their unlinking + } + + if err := notify(ctx, config, cert); err != nil { + return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.LeafSHA256, err) + } + + if notifiedPath != "" { + if err := os.WriteFile(notifiedPath, nil, 0666); err != nil { + return fmt.Errorf("error saving certificate %x: %w", cert.LeafSHA256, err) + } + } + + return nil +} + +func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error { + // TODO-4: save the malformed entry (in get-entries format) in the state directory so user can inspect it + + malformed := &malformedLogEntry{ + Entry: entry, + Error: parseError.Error(), + } + if err := notify(ctx, config, malformed); err != nil { + return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err) + } + return nil +} diff --git a/monitor/statedir.go b/monitor/statedir.go new file mode 100644 index 0000000..a6c9fe3 --- /dev/null +++ b/monitor/statedir.go @@ -0,0 +1,155 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/merkletree" + "strconv" + "strings" + "time" +) + +func readVersion(stateDir string) (int, error) { + path := filepath.Join(stateDir, "version") + + fileBytes, err := os.ReadFile(path) + if errors.Is(err, fs.ErrNotExist) { + if fileExists(filepath.Join(stateDir, "evidence")) { + return 0, nil + } else { + return -1, nil + } + } else if err != nil { + return -1, err + } + + version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes))) + if err != nil { + return -1, fmt.Errorf("version file %q is malformed: %w", path, err) + } + + return version, nil +} + +func writeVersion(stateDir string) error { + return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666) +} + +func migrateLogStateDirV1(dir string) error { + var sth ct.SignedTreeHead + var tree merkletree.CollapsedTree + + sthPath := filepath.Join(dir, "sth.json") + sthData, err := os.ReadFile(sthPath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + treePath := filepath.Join(dir, "tree.json") + treeData, err := os.ReadFile(treePath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } else if err != nil { + return err + } + + if err := json.Unmarshal(sthData, &sth); err != nil { + return fmt.Errorf("error unmarshaling %s: %w", sthPath, err) + } + if err := json.Unmarshal(treeData, &tree); err != nil { + return fmt.Errorf("error unmarshaling %s: %w", treePath, err) + } + + stateFile := stateFile{ + DownloadPosition: &tree, + VerifiedPosition: &tree, + VerifiedSTH: &sth, + LastSuccess: time.Now().UTC(), + } + if stateFile.store(filepath.Join(dir, "state.json")); err != nil { + return err + } + + if err := os.Remove(sthPath); err != nil { + return err + } + if err := os.Remove(treePath); err != nil { + return err + } + return nil +} + +func migrateStateDirV1(stateDir string) error { + if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) { + return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile) + } + + if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil { + for _, logDir := range logDirs { + if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() { + continue + } + if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil { + return fmt.Errorf("error migrating log state: %w", err) + } + } + } else if !errors.Is(err, fs.ErrNotExist) { + return err + } + + if err := writeVersion(stateDir); err != nil { + return err + } + + if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + + return nil +} + +func prepareStateDir(stateDir string) error { + if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + + if version, err := readVersion(stateDir); err != nil { + return err + } else if version == -1 { + if err := writeVersion(stateDir); err != nil { + return err + } + } else if version == 0 { + return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir) + } else if version == 1 { + if err := migrateStateDirV1(stateDir); err != nil { + return err + } + } else if version > 2 { + return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir) + } + + for _, subdir := range []string{"certs", "logs"} { + if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) { + return err + } + } + + return nil +} diff --git a/monitor/statefile.go b/monitor/statefile.go new file mode 100644 index 0000000..c36908e --- /dev/null +++ b/monitor/statefile.go @@ -0,0 +1,47 @@ +// Copyright (C) 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "encoding/json" + "fmt" + "os" + "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/merkletree" + "time" +) + +type stateFile struct { + DownloadPosition *merkletree.CollapsedTree `json:"download_position"` + VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"` + VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"` + LastSuccess time.Time `json:"last_success"` +} + +func loadStateFile(filePath string) (*stateFile, error) { + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + file := new(stateFile) + if err := json.Unmarshal(fileBytes, file); err != nil { + return nil, fmt.Errorf("error parsing %s: %w", filePath, err) + } + return file, nil +} + +func (file *stateFile) store(filePath string) error { + fileBytes, err := json.Marshal(file) + if err != nil { + return err + } + fileBytes = append(fileBytes, '\n') + return writeFile(filePath, fileBytes, 0666) +} diff --git a/monitor/sthdir.go b/monitor/sthdir.go new file mode 100644 index 0000000..9249f13 --- /dev/null +++ b/monitor/sthdir.go @@ -0,0 +1,96 @@ +// Copyright (C) 2017, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "golang.org/x/exp/slices" + "io/fs" + "os" + "path/filepath" + "software.sslmate.com/src/certspotter/ct" + "strconv" + "strings" +) + +func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) { + entries, err := os.ReadDir(dirPath) + if errors.Is(err, fs.ErrNotExist) { + return []*ct.SignedTreeHead{}, nil + } else if err != nil { + return nil, err + } + sths := make([]*ct.SignedTreeHead, 0, len(entries)) + for _, entry := range entries { + filename := entry.Name() + if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") { + continue + } + sth, err := readSTHFile(filepath.Join(dirPath, filename)) + if err != nil { + return nil, err + } + sths = append(sths, sth) + } + slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) bool { return a.TreeSize < b.TreeSize }) + return sths, nil +} + +func readSTHFile(filePath string) (*ct.SignedTreeHead, error) { + fileBytes, err := os.ReadFile(filePath) + if err != nil { + return nil, err + } + sth := new(ct.SignedTreeHead) + if err := json.Unmarshal(fileBytes, sth); err != nil { + return nil, fmt.Errorf("error parsing %s: %w", filePath, err) + } + return sth, nil +} + +func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error { + filePath := filepath.Join(dirPath, sthFilename(sth)) + if fileExists(filePath) { + return nil + } + fileBytes, err := json.Marshal(sth) + if err != nil { + return err + } + return writeFile(filePath, fileBytes, 0666) +} + +func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error { + filePath := filepath.Join(dirPath, sthFilename(sth)) + err := os.Remove(filePath) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + return nil +} + +// generate a filename that uniquely identifies the STH (within the context of a particular log) +func sthFilename(sth *ct.SignedTreeHead) string { + hasher := sha256.New() + switch sth.Version { + case ct.V1: + binary.Write(hasher, binary.LittleEndian, sth.Timestamp) + binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash) + default: + panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version)) + } + // For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic) + return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json" +} diff --git a/monitor/watchlist.go b/monitor/watchlist.go new file mode 100644 index 0000000..8a80852 --- /dev/null +++ b/monitor/watchlist.go @@ -0,0 +1,135 @@ +// Copyright (C) 2016, 2023 Opsmate, Inc. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License, v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. +// +// This software is distributed WITHOUT A WARRANTY OF ANY KIND. +// See the Mozilla Public License for details. + +package monitor + +import ( + "bufio" + "fmt" + "golang.org/x/net/idna" + "io" + "software.sslmate.com/src/certspotter" + "strings" +) + +type WatchItem struct { + domain []string + acceptSuffix bool +} + +type WatchList []WatchItem + +func ParseWatchItem(str string) (WatchItem, error) { + fields := strings.Fields(str) + if len(fields) == 0 { + return WatchItem{}, fmt.Errorf("empty domain") + } + domain := fields[0] + + for _, field := range fields[1:] { + switch { + case strings.HasPrefix(field, "valid_at:"): + // Ignore for backwards compatibility + default: + return WatchItem{}, fmt.Errorf("unknown parameter %q", field) + } + } + + if domain == "." { + // "." as in root zone -> matches everything + return WatchItem{ + domain: []string{}, + acceptSuffix: true, + }, nil + } + + acceptSuffix := false + if strings.HasPrefix(domain, ".") { + acceptSuffix = true + domain = domain[1:] + } + + asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, "."))) + if err != nil { + return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err) + } + return WatchItem{ + domain: strings.Split(asciiDomain, "."), + acceptSuffix: acceptSuffix, + }, nil +} + +func ReadWatchList(reader io.Reader) (WatchList, error) { + items := make(WatchList, 0, 50) + scanner := bufio.NewScanner(reader) + lineNo := 0 + for scanner.Scan() { + line := scanner.Text() + lineNo++ + if line == "" || strings.HasPrefix(line, "#") { + continue + } + item, err := ParseWatchItem(line) + if err != nil { + return nil, fmt.Errorf("%w on line %d", err, lineNo) + } + items = append(items, item) + } + return items, scanner.Err() +} + +func (item WatchItem) String() string { + if item.acceptSuffix { + return "." + strings.Join(item.domain, ".") + } else { + return strings.Join(item.domain, ".") + } +} + +func (item WatchItem) matchesDNSName(dnsName []string) bool { + watchDomain := item.domain + for len(dnsName) > 0 && len(watchDomain) > 0 { + certLabel := dnsName[len(dnsName)-1] + watchLabel := watchDomain[len(watchDomain)-1] + + if !dnsLabelMatches(certLabel, watchLabel) { + return false + } + + dnsName = dnsName[:len(dnsName)-1] + watchDomain = watchDomain[:len(watchDomain)-1] + } + return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0) +} + +func dnsLabelMatches(certLabel string, watchLabel string) bool { + // For fail-safe behavior, if a label was unparsable, it matches everything. + // Similarly, redacted labels match everything, since the label _might_ be + // for a name we're interested in. + + return certLabel == "*" || + certLabel == "?" || + certLabel == certspotter.UnparsableDNSLabelPlaceholder || + certspotter.MatchesWildcard(watchLabel, certLabel) +} + +func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) { + dnsNames := make([][]string, len(identifiers.DNSNames)) + for i, dnsName := range identifiers.DNSNames { + dnsNames[i] = strings.Split(dnsName, ".") + } + for _, item := range list { + for _, dnsName := range dnsNames { + if item.matchesDNSName(dnsName) { + return true, item + } + } + } + return false, WatchItem{} +}