Tandem
/
certspotter
mirror of https://github.com/SSLMate/certspotter.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert to a daemon and make many other improvements 

Specifically, certspotter no longer terminates unless it receives SIGTERM
or SIGINT or there is a serious error.

Although using cron made sense in the early days of Certificate
Transparency, certspotter now needs to run continuously to reliably keep
up with the high growth rate of contemporary CT logs, and to gracefully
handle the many transient errors that can arise when monitoring CT.

Closes: #63
Closes: #37
Closes: #32 (presumably by eliminating $DNS_NAMES and $IP_ADDRESSES)
Closes: #21 (with $WATCH_ITEM)
Closes: #25
This commit is contained in:
Andrew Ayer 2023-02-03 14:05:04 -05:00
parent e3835dea53
commit 209cdb181b
18 changed files with 1775 additions and 190 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

319
cmd/certspotter/main.go
View File

@ -1,4 +1,4 @@
// Copyright (C) 2016 Opsmate, Inc.
// Copyright (C) 2016, 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
@ -10,224 +10,169 @@
package main
import (
"bufio"
"context"
"errors"
"flag"
"fmt"
"io"
"io/fs"
insecurerand "math/rand"
"os"
"os/signal"
"path/filepath"
"runtime/debug"
"strings"
"syscall"
"time"
"golang.org/x/net/idna"
"software.sslmate.com/src/certspotter"
"software.sslmate.com/src/certspotter/cmd"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/monitor"
)
var programName = os.Args[0]
const defaultLogList = "https://loglist.certspotter.org/monitor.json"
func certspotterVersion() string {
info, ok := debug.ReadBuildInfo()
if !ok {
return "unknown"
}
if strings.HasPrefix(info.Main.Version, "v") {
return info.Main.Version
}
var vcs, vcsRevision, vcsModified string
for _, s := range info.Settings {
switch s.Key {
case "vcs":
vcs = s.Value
case "vcs.revision":
vcsRevision = s.Value
case "vcs.modified":
vcsModified = s.Value
}
}
if vcs == "git" && vcsRevision != "" && vcsModified == "true" {
return vcsRevision + "+"
} else if vcs == "git" && vcsRevision != "" {
return vcsRevision
}
return "unknown"
}
func homedir() string {
homedir, err := os.UserHomeDir()
if err != nil {
panic(fmt.Errorf("unable to determine home directory: %w", err))
}
return homedir
}
func defaultStateDir() string {
if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" {
return envVar
} else {
return cmd.DefaultStateDir("certspotter")
return filepath.Join(homedir(), ".certspotter")
}
}
func defaultConfigDir() string {
if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" {
return envVar
} else {
return cmd.DefaultConfigDir("certspotter")
return filepath.Join(homedir(), ".certspotter")
}
}
func trimTrailingDots(value string) string {
length := len(value)
for length > 0 && value[length-1] == '.' {
length--
}
return value[0:length]
}
var stateDir = flag.String("state_dir", defaultStateDir(), "Directory for storing state")
var watchlistFilename = flag.String("watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing identifiers to watch (- for stdin)")
type watchlistItem struct {
Domain []string
AcceptSuffix bool
ValidAt *time.Time // optional
}
var watchlist []watchlistItem
func parseWatchlistItem(str string) (watchlistItem, error) {
fields := strings.Fields(str)
if len(fields) == 0 {
return watchlistItem{}, fmt.Errorf("Empty domain")
}
domain := fields[0]
var validAt *time.Time = nil
// parse options
for i := 1; i < len(fields); i++ {
chunks := strings.SplitN(fields[i], ":", 2)
if len(chunks) != 2 {
return watchlistItem{}, fmt.Errorf("Missing Value `%s'", fields[i])
}
switch chunks[0] {
case "valid_at":
validAtTime, err := time.Parse("2006-01-02", chunks[1])
if err != nil {
return watchlistItem{}, fmt.Errorf("Invalid Date `%s': %s", chunks[1], err)
}
validAt = &validAtTime
default:
return watchlistItem{}, fmt.Errorf("Unknown Option `%s'", fields[i])
}
}
// parse domain
// "." as in root zone (matches everything)
if domain == "." {
return watchlistItem{
Domain: []string{},
AcceptSuffix: true,
ValidAt: validAt,
}, nil
}
acceptSuffix := false
if strings.HasPrefix(domain, ".") {
acceptSuffix = true
domain = domain[1:]
}
asciiDomain, err := idna.ToASCII(strings.ToLower(trimTrailingDots(domain)))
func readWatchListFile(filename string) (monitor.WatchList, error) {
file, err := os.Open(filename)
if err != nil {
return watchlistItem{}, fmt.Errorf("Invalid domain `%s': %s", domain, err)
}
return watchlistItem{
Domain: strings.Split(asciiDomain, "."),
AcceptSuffix: acceptSuffix,
ValidAt: validAt,
}, nil
}
func readWatchlist(reader io.Reader) ([]watchlistItem, error) {
items := []watchlistItem{}
scanner := bufio.NewScanner(reader)
for scanner.Scan() {
line := scanner.Text()
if line == "" || strings.HasPrefix(line, "#") {
continue
var pathErr *fs.PathError
if errors.As(err, &pathErr) {
err = pathErr.Err
}
item, err := parseWatchlistItem(line)
if err != nil {
return nil, err
}
items = append(items, item)
return nil, err
}
return items, scanner.Err()
defer file.Close()
return monitor.ReadWatchList(file)
}
func dnsLabelMatches(certLabel string, watchLabel string) bool {
// For fail-safe behavior, if a label was unparsable, it matches everything.
// Similarly, redacted labels match everything, since the label _might_ be
// for a name we're interested in.
return certLabel == "*" ||
certLabel == "?" ||
certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
certspotter.MatchesWildcard(watchLabel, certLabel)
}
func dnsNameMatches(dnsName []string, watchDomain []string, acceptSuffix bool) bool {
for len(dnsName) > 0 && len(watchDomain) > 0 {
certLabel := dnsName[len(dnsName)-1]
watchLabel := watchDomain[len(watchDomain)-1]
if !dnsLabelMatches(certLabel, watchLabel) {
return false
}
dnsName = dnsName[:len(dnsName)-1]
watchDomain = watchDomain[:len(watchDomain)-1]
}
return len(watchDomain) == 0 && (acceptSuffix || len(dnsName) == 0)
}
func anyDnsNameIsWatched(info *certspotter.EntryInfo) bool {
dnsNames := info.Identifiers.DNSNames
matched := false
for _, dnsName := range dnsNames {
labels := strings.Split(dnsName, ".")
for _, item := range watchlist {
if dnsNameMatches(labels, item.Domain, item.AcceptSuffix) {
if item.ValidAt != nil {
// BygoneSSL Check
// was the SSL certificate issued before the domain was registered
// and valid after
if item.ValidAt.Before(*info.CertInfo.NotAfter()) &&
item.ValidAt.After(*info.CertInfo.NotBefore()) {
info.Bygone = true
return true
}
}
// keep iterating in case another domain watched matches valid_at
matched = true
}
}
}
return matched
}
func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) {
info := certspotter.EntryInfo{
LogUri: scanner.LogUri,
Entry: entry,
IsPrecert: certspotter.IsPrecert(entry),
FullChain: certspotter.GetFullChain(entry),
}
info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry)
if info.CertInfo != nil {
info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers()
}
// Fail safe behavior: if info.Identifiers is nil (which is caused by a
// parse error), report the certificate because we can't say for sure it
// doesn't match a domain we care about. We try very hard to make sure
// parsing identifiers always succeeds, so false alarms should be rare.
if info.Identifiers == nil || anyDnsNameIsWatched(&info) {
cmd.LogEntry(&info)
func appendFunc(slice *[]string) func(string) error {
return func(value string) error {
*slice = append(*slice, value)
return nil
}
}
func main() {
cmd.ParseFlags()
insecurerand.Seed(time.Now().UnixNano()) // TODO: remove after upgrading to Go 1.20
if *watchlistFilename == "-" {
var err error
watchlist, err = readWatchlist(os.Stdin)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: (stdin): %s\n", os.Args[0], err)
os.Exit(1)
}
} else {
file, err := os.Open(*watchlistFilename)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
os.Exit(1)
}
watchlist, err = readWatchlist(file)
file.Close()
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
os.Exit(1)
}
// TODO-3: set loglist.UserAgent
var flags struct {
batchSize int // TODO-4: respect this option
email []string
logs string
noSave bool
script string
startAtEnd bool
stateDir string
stdout bool
verbose bool
version bool
watchlist string
}
flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)")
flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email))
flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)")
flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
flag.BoolVar(&flags.version, "version", false, "Print version and exit")
flag.StringVar(&flags.watchlist, "watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing domain names to watch")
flag.Parse()
if flags.version {
fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
os.Exit(0)
}
os.Exit(cmd.Main(*stateDir, processEntry))
if len(flags.email) == 0 && len(flags.script) == 0 && flags.stdout == false {
fmt.Fprintf(os.Stderr, "%s: at least one of -email, -script, or -stdout must be specified (see -help for details)\n", programName)
os.Exit(2)
}
config := &monitor.Config{
LogListSource: flags.logs,
StateDir: flags.stateDir,
SaveCerts: !flags.noSave,
StartAtEnd: flags.startAtEnd,
Verbose: flags.verbose,
Script: flags.script,
Email: flags.email,
Stdout: flags.stdout,
}
if flags.watchlist == "-" {
watchlist, err := monitor.ReadWatchList(os.Stdin)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err)
os.Exit(1)
}
config.WatchList = watchlist
} else {
watchlist, err := readWatchListFile(flags.watchlist)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err)
os.Exit(1)
}
config.WatchList = watchlist
}
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) {
fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
os.Exit(1)
}
}

8
go.mod
View File

@ -1,8 +1,10 @@
module software.sslmate.com/src/certspotter
go 1.17
go 1.19
require (
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
golang.org/x/text v0.3.7 // indirect
golang.org/x/exp v0.0.0-20230118134722-a68e582fa157 // indirect
golang.org/x/net v0.5.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/text v0.6.0 // indirect
)

8
go.sum
View File

@ -1,4 +1,12 @@
golang.org/x/exp v0.0.0-20230118134722-a68e582fa157 h1:fiNkyhJPUvxbRPbCqY/D9qdjmPzfHcpK3P4bM4gioSY=
golang.org/x/exp v0.0.0-20230118134722-a68e582fa157/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=

22
monitor/config.go Normal file
View File

@ -0,0 +1,22 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
type Config struct {
LogListSource string
StateDir string
StartAtEnd bool
WatchList WatchList
Verbose bool
SaveCerts bool
Script string
Email []string
Stdout bool
}

142
monitor/daemon.go Normal file
View File

@ -0,0 +1,142 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"context"
"errors"
"fmt"
"golang.org/x/sync/errgroup"
"log"
insecurerand "math/rand"
"software.sslmate.com/src/certspotter/loglist"
"time"
)
const (
reloadLogListIntervalMin = 3 * time.Second // TODO-3: use 30 * time.Minute
reloadLogListIntervalMax = 9 * time.Second // TODO-3: use 90 * time.Minute
healthCheckInterval = 24 * time.Hour
)
func randomDuration(min, max time.Duration) time.Duration {
return min + time.Duration(insecurerand.Int63n(int64(max-min+1)))
}
func reloadLogListInterval() time.Duration {
return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax)
}
type task struct {
stop context.CancelFunc
}
type daemon struct {
config *Config
taskgroup *errgroup.Group
tasks map[LogID]task
logsLoadedAt time.Time
}
func (daemon *daemon) healthCheck(ctx context.Context) error { // TODO-2
return nil
}
func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
ctx, cancel := context.WithCancel(ctx)
daemon.taskgroup.Go(func() error {
defer cancel()
err := monitorLogContinously(ctx, daemon.config, ctlog)
if daemon.config.Verbose {
log.Printf("task for log %s stopped with error %s", ctlog.URL, err)
}
if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
return nil
} else {
return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err)
}
})
return task{stop: cancel}
}
func (daemon *daemon) loadLogList(ctx context.Context) error {
loglist, err := getLogList(ctx, daemon.config.LogListSource)
if err != nil {
return err
}
if daemon.config.Verbose {
log.Printf("fetched %d logs from %q", len(loglist), daemon.config.LogListSource)
}
for logID, task := range daemon.tasks {
if _, exists := loglist[logID]; exists {
continue
}
if daemon.config.Verbose {
log.Printf("stopping task for log %s", logID.Base64String())
}
task.stop()
delete(daemon.tasks, logID)
}
for logID, ctlog := range loglist {
if _, isRunning := daemon.tasks[logID]; isRunning {
continue
}
if daemon.config.Verbose {
log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL)
}
daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
}
daemon.logsLoadedAt = time.Now()
return nil
}
func (daemon *daemon) run(ctx context.Context) error {
if err := prepareStateDir(daemon.config.StateDir); err != nil {
return fmt.Errorf("error preparing state directory: %w", err)
}
if err := daemon.loadLogList(ctx); err != nil {
return fmt.Errorf("error loading log list: %w", err)
}
reloadLogListTicker := time.NewTicker(reloadLogListInterval())
defer reloadLogListTicker.Stop()
healthCheckTicker := time.NewTicker(healthCheckInterval)
defer healthCheckTicker.Stop()
for ctx.Err() == nil {
select {
case <-ctx.Done():
case <-reloadLogListTicker.C:
if err := daemon.loadLogList(ctx); err != nil {
recordError(fmt.Errorf("error reloading log list (will try again later): %w", err))
}
reloadLogListTicker.Reset(reloadLogListInterval())
case <-healthCheckTicker.C:
if err := daemon.healthCheck(ctx); err != nil {
return err
}
}
}
return ctx.Err()
}
func Run(ctx context.Context, config *Config) error {
group, ctx := errgroup.WithContext(ctx)
daemon := &daemon{
config: config,
taskgroup: group,
tasks: make(map[LogID]task),
}
group.Go(func() error { return daemon.run(ctx) })
return group.Wait()
}

188
monitor/discoveredcert.go Normal file
View File

@ -0,0 +1,188 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/pem"
"fmt"
"strings"
"time"
"software.sslmate.com/src/certspotter"
"software.sslmate.com/src/certspotter/ct"
)
type discoveredCert struct {
WatchItem WatchItem
LogEntry *logEntry
Info *certspotter.CertInfo
Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate
LeafSHA256 [32]byte // computed over Chain[0]
Identifiers *certspotter.Identifiers
CertPath string // empty if not saved on the filesystem
JSONPath string // empty if not saved on the filesystem
TextPath string // empty if not saved on the filesystem
}
func (cert *discoveredCert) pemChain() []byte {
var buffer bytes.Buffer
for _, certBytes := range cert.Chain {
if err := pem.Encode(&buffer, &pem.Block{
Type: "CERTIFICATE",
Bytes: certBytes,
}); err != nil {
panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err))
}
}
return buffer.Bytes()
}
func (cert *discoveredCert) json() []byte {
pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey())
object := map[string]any{
"cert_sha256": hex.EncodeToString(cert.LeafSHA256[:]),
"pubkey_sha256": hex.EncodeToString(pubkeySha256[:]),
"issuer_der": cert.Info.TBS.Issuer.FullBytes,
"subject_der": cert.Info.TBS.Subject.FullBytes,
"dns_names": cert.Identifiers.DNSNames,
"ip_addresses": cert.Identifiers.IPAddrs,
}
if cert.Info.ValidityParseError == nil {
object["not_before"] = cert.Info.Validity.NotBefore
object["not_after"] = cert.Info.Validity.NotAfter
} else {
object["not_before"] = nil
object["not_after"] = nil
}
if cert.Info.SerialNumberParseError == nil {
object["serial_number"] = fmt.Sprintf("%x", cert.Info.SerialNumber)
} else {
object["serial_number"] = nil
}
jsonBytes, err := json.Marshal(object)
if err != nil {
panic(fmt.Errorf("encoding certificate as JSON failed unexpectedly: %w", err))
}
return jsonBytes
}
func (cert *discoveredCert) save() error {
if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil {
return err
}
if err := writeFile(cert.JSONPath, cert.json(), 0666); err != nil {
return err
}
if err := writeFile(cert.TextPath, []byte(cert.Text()), 0666); err != nil {
return err
}
return nil
}
func (cert *discoveredCert) Environ() []string {
pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey())
env := []string{
"EVENT=discovered_cert",
"SUMMARY=certificate discovered for " + cert.WatchItem.String(),
"CERT_PARSEABLE=yes", // backwards compat; not documented (TODO-3: consider removing)
"LOG_URI=" + cert.LogEntry.Log.URL,
"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
"WATCH_ITEM=" + cert.WatchItem.String(),
"CERT_SHA256=" + hex.EncodeToString(cert.LeafSHA256[:]),
"FINGERPRINT=" + hex.EncodeToString(cert.LeafSHA256[:]), // backwards compat; not documented (TODO-3: consider removing)
"PUBKEY_SHA256=" + hex.EncodeToString(pubkeySha256[:]),
"PUBKEY_HASH=" + hex.EncodeToString(pubkeySha256[:]), // backwards compat; not documented (TODO-3: consider removing)
"CERT_FILENAME=" + cert.CertPath,
"JSON_FILENAME=" + cert.JSONPath,
"TEXT_FILENAME=" + cert.TextPath,
}
if cert.Info.ValidityParseError == nil {
env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String())
env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix()))
env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339))
env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String())
env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix()))
env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339))
} else {
env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error())
}
if cert.Info.SubjectParseError == nil {
env = append(env, "SUBJECT_DN="+cert.Info.Subject.String())
} else {
env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error())
}
if cert.Info.IssuerParseError == nil {
env = append(env, "ISSUER_DN="+cert.Info.Issuer.String())
} else {
env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error())
}
// TODO-3: consider removing $SERIAL due to misuse potential, or renaming to $SERIAL_NUMBER
if cert.Info.SerialNumberParseError == nil {
env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber))
} else {
env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
}
return env
}
func (cert *discoveredCert) Text() string {
// TODO-3: improve the output: include WatchItem, indicate hash algorithm. look at sslmate email for inspiration
text := new(strings.Builder)
writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey())
fmt.Fprintf(text, "%x:\n", cert.LeafSHA256)
for _, dnsName := range cert.Identifiers.DNSNames {
writeField("DNS Name", dnsName)
}
for _, ipaddr := range cert.Identifiers.IPAddrs {
writeField("IP Address", ipaddr)
}
writeField("Pubkey", hex.EncodeToString(pubkeySha256[:]))
if cert.Info.IssuerParseError == nil {
writeField("Issuer", cert.Info.Issuer)
} else {
writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError))
}
if cert.Info.ValidityParseError == nil {
writeField("Not Before", cert.Info.Validity.NotBefore)
writeField("Not After", cert.Info.Validity.NotAfter)
} else {
writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
}
writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL)) // TODO-3: include entry type here?
writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.LeafSHA256[:]))
if cert.CertPath != "" {
writeField("Filename", cert.CertPath)
}
return text.String()
}
func (cert *discoveredCert) EmailSubject() string {
return fmt.Sprintf("[certspotter] Certificate Discovered for %s", cert.WatchItem)
}

18
monitor/errors.go Normal file
View File

@ -0,0 +1,18 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"log"
)
func recordError(err error) {
log.Print(err)
}

42
monitor/fileutils.go Normal file
View File

@ -0,0 +1,42 @@
// Copyright (C) 2017, 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"crypto/rand"
"encoding/hex"
"fmt"
"os"
)
func randomFileSuffix() string {
var randomBytes [12]byte
if _, err := rand.Read(randomBytes[:]); err != nil {
panic(err)
}
return hex.EncodeToString(randomBytes[:])
}
func writeFile(filename string, data []byte, perm os.FileMode) error {
tempname := filename + ".tmp." + randomFileSuffix()
if err := os.WriteFile(tempname, data, perm); err != nil {
return fmt.Errorf("error writing %s: %w", filename, err)
}
if err := os.Rename(tempname, filename); err != nil {
os.Remove(tempname)
return fmt.Errorf("error writing %s: %w", filename, err)
}
return nil
}
func fileExists(filename string) bool {
_, err := os.Lstat(filename)
return err == nil
}

102
monitor/healthcheck.go Normal file
View File

@ -0,0 +1,102 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"fmt"
"strings"
"time"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/loglist"
)
type staleSTHEvent struct {
Log *loglist.Log
LastSuccess time.Time
LatestSTH *ct.SignedTreeHead // may be nil
}
type backlogEvent struct {
Log *loglist.Log
LatestSTH *ct.SignedTreeHead
Backlog uint64
Position uint64
}
type staleLogListEvent struct {
Source string
LastSuccess time.Time
LastError string
LastErrorTime time.Time
}
func (e *staleSTHEvent) Environ() []string {
return []string{
"EVENT=error",
"SUMMARY=" + fmt.Sprintf("unable to contact %s since %s", e.Log.URL, e.LastSuccess),
}
}
func (e *backlogEvent) Environ() []string {
return []string{
"EVENT=error",
"SUMMARY=" + fmt.Sprintf("backlog of size %d from %s", e.Backlog, e.Log.URL),
}
}
func (e *staleLogListEvent) Environ() []string {
return []string{
"EVENT=error",
"SUMMARY=" + fmt.Sprintf("unable to retrieve log list since %s: %s", e.LastSuccess, e.LastError),
}
}
func (e *staleSTHEvent) EmailSubject() string {
return fmt.Sprintf("[certspotter] Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
}
func (e *backlogEvent) EmailSubject() string {
return fmt.Sprintf("[certspotter] Backlog of size %d from %s", e.Backlog, e.Log.URL)
}
func (e *staleLogListEvent) EmailSubject() string {
return fmt.Sprintf("[certspotter] Unable to retrieve log list since %s", e.LastSuccess)
}
func (e *staleSTHEvent) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")
if e.LatestSTH != nil {
fmt.Fprintf(text, "Latest known log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.Timestamp)
} else {
fmt.Fprintf(text, "Latest known log size = none\n")
}
return text.String()
}
func (e *backlogEvent) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.Timestamp)
fmt.Fprintf(text, "Current position = %d\n", e.Position)
fmt.Fprintf(text, " Backlog = %d\n", e.Backlog)
return text.String()
}
func (e *staleLogListEvent) Text() string {
text := new(strings.Builder)
fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError)
fmt.Fprintf(text, "\n")
fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n")
return text.String()
}
// TODO-3: make the errors more actionable

40
monitor/loglist.go Normal file
View File

@ -0,0 +1,40 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"context"
"fmt"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/loglist"
)
type LogID = ct.SHA256Hash
func getLogList(ctx context.Context, source string) (map[LogID]*loglist.Log, error) {
// TODO-4: pass context to loglist.Load
// TODO-3: If-Modified-Since / If-None-Match support
list, err := loglist.Load(source)
if err != nil {
return nil, err
}
logs := make(map[LogID]*loglist.Log)
for operatorIndex := range list.Operators {
for logIndex := range list.Operators[operatorIndex].Logs {
log := &list.Operators[operatorIndex].Logs[logIndex]
if _, exists := logs[log.LogID]; exists {
return nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
}
logs[log.LogID] = log
}
}
return logs, nil
}

48
monitor/malformed.go Normal file
View File

@ -0,0 +1,48 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"fmt"
"strings"
)
type malformedLogEntry struct {
Entry *logEntry
Error string
}
func (malformed *malformedLogEntry) Environ() []string {
return []string{
"EVENT=discovered_cert",
"SUMMARY=" + fmt.Sprintf("unable to parse entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL),
"LOG_URI=" + malformed.Entry.Log.URL,
"ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index),
"LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(),
"PARSE_ERROR=" + malformed.Error,
"CERT_PARSEABLE=no", // backwards compat; not documented (TODO-3: consider removing)
}
}
func (malformed *malformedLogEntry) Text() string {
text := new(strings.Builder)
writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL))
writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String())
writeField("Error", malformed.Error)
return text.String()
}
func (malformed *malformedLogEntry) EmailSubject() string {
return fmt.Sprintf("[certspotter] Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL)
}

294
monitor/monitor.go Normal file
View File

@ -0,0 +1,294 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"context"
"crypto/x509"
"errors"
"fmt"
"io/fs"
"log"
"os"
"path/filepath"
"strings"
"time"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/ct/client"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/merkletree"
)
const (
maxGetEntriesSize = 1000
//monitorLogInterval = 5 * time.Minute // TODO-3
monitorLogInterval = 15 * time.Second
)
func isFatalLogError(err error) bool {
return errors.Is(err, context.Canceled)
}
func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) {
logKey, err := x509.ParsePKIXPublicKey(ctlog.Key)
if err != nil {
return nil, fmt.Errorf("error parsing log key: %w", err)
}
verifier, err := ct.NewSignatureVerifier(logKey)
if err != nil {
return nil, fmt.Errorf("error with log key: %w", err)
}
return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil
}
func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error {
logClient, err := newLogClient(ctlog)
if err != nil {
return err
}
ticker := time.NewTicker(monitorLogInterval)
defer ticker.Stop()
for ctx.Err() == nil {
if err := monitorLog(ctx, config, ctlog, logClient); err != nil {
return err
}
select {
case <-ctx.Done():
case <-ticker.C:
}
}
return ctx.Err()
}
func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
var (
stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
stateFilePath = filepath.Join(stateDirPath, "state.json")
sthsDirPath = filepath.Join(stateDirPath, "unverified_sths")
)
for _, dirPath := range []string{stateDirPath, sthsDirPath} {
if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("error creating state directory: %w", err)
}
}
startTime := time.Now()
latestSTH, err := logClient.GetSTH(ctx)
if isFatalLogError(err) {
return err
} else if err != nil {
recordError(fmt.Errorf("error fetching latest STH for %s: %w", ctlog.URL, err))
return nil
}
latestSTH.LogID = ctlog.LogID
if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil {
return fmt.Errorf("error storing latest STH: %w", err)
}
state, err := loadStateFile(stateFilePath)
if errors.Is(err, fs.ErrNotExist) {
if config.StartAtEnd {
tree, err := reconstructTree(ctx, logClient, latestSTH)
if isFatalLogError(err) {
return err
} else if err != nil {
recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err))
return nil
}
state = &stateFile{
DownloadPosition: tree,
VerifiedPosition: tree,
VerifiedSTH: latestSTH,
LastSuccess: startTime.UTC(),
}
} else {
state = &stateFile{
DownloadPosition: merkletree.EmptyCollapsedTree(),
VerifiedPosition: merkletree.EmptyCollapsedTree(),
VerifiedSTH: nil,
LastSuccess: startTime.UTC(),
}
}
if config.Verbose {
log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
}
if err := state.store(stateFilePath); err != nil {
return fmt.Errorf("error storing state file: %w", err)
}
} else if err != nil {
return fmt.Errorf("error loading state file: %w", err)
}
sths, err := loadSTHsFromDir(sthsDirPath)
if err != nil {
return fmt.Errorf("error loading STHs directory: %w", err)
}
for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
// TODO-4: audit sths[0] against state.VerifiedSTH
if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
return fmt.Errorf("error removing STH: %w", err)
}
sths = sths[1:]
}
defer func() {
if config.Verbose {
log.Printf("saving state in defer for %s", ctlog.URL)
}
if err := state.store(stateFilePath); err != nil && returnedErr == nil {
returnedErr = fmt.Errorf("error storing state file: %w", err)
}
}()
if len(sths) == 0 {
state.LastSuccess = startTime.UTC()
return nil
}
var (
downloadBegin = state.DownloadPosition.Size()
downloadEnd = sths[len(sths)-1].TreeSize
entries = make(chan client.GetEntriesItem, maxGetEntriesSize)
downloadErr error
)
if config.Verbose {
log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd)
}
go func() {
defer close(entries)
downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
}()
for rawEntry := range entries {
entry := &logEntry{
Log: ctlog,
Index: state.DownloadPosition.Size(),
LeafInput: rawEntry.LeafInput,
ExtraData: rawEntry.ExtraData,
LeafHash: merkletree.HashLeaf(rawEntry.LeafInput),
}
if err := processLogEntry(ctx, config, entry); err != nil {
return fmt.Errorf("error processing entry %d: %w", entry.Index, err)
}
state.DownloadPosition.Add(entry.LeafHash)
rootHash := state.DownloadPosition.CalculateRoot()
shouldSaveState := state.DownloadPosition.Size()%10000 == 0
for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
state.DownloadPosition = state.VerifiedPosition
if err := state.store(stateFilePath); err != nil {
return fmt.Errorf("error storing state file: %w", err)
}
return nil
}
state.VerifiedPosition = state.DownloadPosition
state.VerifiedSTH = sths[0]
shouldSaveState = true
if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
return fmt.Errorf("error removing verified STH: %w", err)
}
sths = sths[1:]
}
if shouldSaveState {
if err := state.store(stateFilePath); err != nil {
return fmt.Errorf("error storing state file: %w", err)
}
}
}
if isFatalLogError(downloadErr) {
return downloadErr
} else if downloadErr != nil {
recordError(fmt.Errorf("error downloading entries from %s: %w", ctlog.URL, downloadErr))
return nil
}
if config.Verbose {
log.Printf("finished downloading entries from %s", ctlog.URL)
}
state.LastSuccess = startTime.UTC()
return nil
}
func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
for begin < end && ctx.Err() == nil {
size := begin - end
if size > maxGetEntriesSize {
size = maxGetEntriesSize
}
entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1)
if err != nil {
return err
}
for _, entry := range entries {
if ctx.Err() != nil {
return ctx.Err()
}
select {
case <-ctx.Done():
return ctx.Err()
case entriesChan <- entry:
}
}
begin += uint64(len(entries))
}
return ctx.Err()
}
func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) {
if sth.TreeSize == 0 {
return merkletree.EmptyCollapsedTree(), nil
}
entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1)
if err != nil {
return nil, err
}
leafHash := merkletree.HashLeaf(entries[0].LeafInput)
var tree *merkletree.CollapsedTree
if sth.TreeSize > 1 {
auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize)
if err != nil {
return nil, err
}
hashes := make([]merkletree.Hash, len(auditPath))
for i := range hashes {
copy(hashes[i][:], auditPath[len(auditPath)-i-1])
}
tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1)
if err != nil {
return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err)
}
} else {
tree = merkletree.EmptyCollapsedTree()
}
tree.Add(leafHash)
rootHash := tree.CalculateRoot()
if rootHash != merkletree.Hash(sth.SHA256RootHash) {
return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize)
}
return tree, nil
}

151
monitor/notify.go Normal file
View File

@ -0,0 +1,151 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"bytes"
"context"
"errors"
"fmt"
"io/fs"
"os"
"os/exec"
"path/filepath"
"strings"
"sync"
)
var stdoutMu sync.Mutex
type notification interface {
Environ() []string
EmailSubject() string
Text() string
}
func notify(ctx context.Context, config *Config, notif notification) error {
if config.Stdout {
writeToStdout(notif)
}
if len(config.Email) > 0 {
if err := sendEmail(ctx, config.Email, notif); err != nil {
return err
}
}
if config.Script != "" {
if err := execScript(ctx, config.Script, notif); err != nil {
return err
}
}
return nil
}
func writeToStdout(notif notification) {
stdoutMu.Lock()
defer stdoutMu.Unlock()
os.Stdout.WriteString(notif.Text() + "\n")
}
func sendEmail(ctx context.Context, to []string, notif notification) error {
stdin := new(bytes.Buffer)
stderr := new(bytes.Buffer)
fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
fmt.Fprintf(stdin, "Subject: %s\n", notif.EmailSubject())
fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
fmt.Fprintf(stdin, "\n")
fmt.Fprint(stdin, notif.Text())
args := []string{"-i", "--"}
args = append(args, to...)
sendmail := exec.CommandContext(ctx, "/usr/sbin/sendmail", args...)
sendmail.Stdin = stdin
sendmail.Stderr = stderr
if err := sendmail.Run(); err == nil {
return nil
} else if ctx.Err() != nil {
return ctx.Err()
} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
} else {
return fmt.Errorf("error sending email to %v: %w", to, err)
}
}
func execScript(ctx context.Context, scriptPath string, notif notification) error {
// TODO-3: consider removing directory support (for now), and supporting $PATH lookups
info, err := os.Stat(scriptPath)
if errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("script %q does not exist", scriptPath)
} else if err != nil {
return fmt.Errorf("error executing script %q: %w", scriptPath, err)
} else if info.IsDir() {
return execScriptDir(ctx, scriptPath, notif)
} else {
return execScriptFile(ctx, scriptPath, notif)
}
}
func execScriptDir(ctx context.Context, dirPath string, notif notification) error {
dirents, err := os.ReadDir(dirPath)
if err != nil {
return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err)
}
for _, dirent := range dirents {
if strings.HasPrefix(dirent.Name(), ".") {
continue
}
scriptPath := filepath.Join(dirPath, dirent.Name())
info, err := os.Stat(scriptPath)
if errors.Is(err, fs.ErrNotExist) {
continue
} else if err != nil {
return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err)
} else if info.Mode().IsRegular() && isExecutable(info.Mode()) {
if err := execScriptFile(ctx, scriptPath, notif); err != nil {
return err
}
}
}
return nil
}
func execScriptFile(ctx context.Context, scriptPath string, notif notification) error {
stderr := new(bytes.Buffer)
cmd := exec.CommandContext(ctx, scriptPath)
cmd.Env = os.Environ()
cmd.Env = append(cmd.Env, notif.Environ()...)
cmd.Stderr = stderr
if err := cmd.Run(); err == nil {
return nil
} else if ctx.Err() != nil {
return ctx.Err()
} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
return fmt.Errorf("script %q exited with code %d and error %q", scriptPath, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
} else if isExitError {
return fmt.Errorf("script %q terminated by signal with error %q", scriptPath, strings.TrimSpace(stderr.String()))
} else {
return fmt.Errorf("error executing script %q: %w", scriptPath, err)
}
}
func isExecutable(mode os.FileMode) bool {
return mode&0111 != 0
}

150
monitor/process.go Normal file
View File

@ -0,0 +1,150 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"software.sslmate.com/src/certspotter"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/loglist"
"software.sslmate.com/src/certspotter/merkletree"
)
type logEntry struct {
Log *loglist.Log
Index uint64
LeafInput []byte
ExtraData []byte
LeafHash merkletree.Hash
}
func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error {
leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
}
switch leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry)
case ct.PrecertLogEntryType:
return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry)
default:
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType))
}
}
func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error {
certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
}
chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err))
}
chain = append([]ct.ASN1Cert{cert}, chain...)
return processCertificate(ctx, config, entry, certInfo, chain)
}
func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error {
certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
}
chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData)
if err != nil {
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err))
}
return processCertificate(ctx, config, entry, certInfo, chain)
}
func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
identifiers, err := certInfo.ParseIdentifiers()
if err != nil {
return processMalformedLogEntry(ctx, config, entry, err)
}
matched, watchItem := config.WatchList.Matches(identifiers)
if !matched {
return nil
}
cert := &discoveredCert{
WatchItem: watchItem,
LogEntry: entry,
Info: certInfo,
Chain: chain,
LeafSHA256: sha256.Sum256(chain[0]),
Identifiers: identifiers,
}
var notifiedPath string
if config.SaveCerts {
hexFingerprint := hex.EncodeToString(cert.LeafSHA256[:])
prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2])
for _, suffix := range []string{".notified", ".cert.pem", ".precert.pem"} {
if fileExists(filepath.Join(prefixPath, hexFingerprint+suffix)) {
return nil
}
}
if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.LeafSHA256, err)
}
notifiedPath = filepath.Join(prefixPath, hexFingerprint+".notified") // TODO-3: maybe this should be a hidden file?
cert.CertPath = filepath.Join(prefixPath, hexFingerprint+".pem")
cert.JSONPath = filepath.Join(prefixPath, hexFingerprint+".json") // TODO-3: consider using .v1.json extension in case I change the format later?
cert.TextPath = filepath.Join(prefixPath, hexFingerprint+".txt")
if err := cert.save(); err != nil {
return fmt.Errorf("error saving certificate %x: %w", cert.LeafSHA256, err)
}
} else {
// TODO-4: save cert to temporary files, and defer their unlinking
}
if err := notify(ctx, config, cert); err != nil {
return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.LeafSHA256, err)
}
if notifiedPath != "" {
if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
return fmt.Errorf("error saving certificate %x: %w", cert.LeafSHA256, err)
}
}
return nil
}
func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error {
// TODO-4: save the malformed entry (in get-entries format) in the state directory so user can inspect it
malformed := &malformedLogEntry{
Entry: entry,
Error: parseError.Error(),
}
if err := notify(ctx, config, malformed); err != nil {
return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
}
return nil
}

155
monitor/statedir.go Normal file
View File

@ -0,0 +1,155 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/merkletree"
"strconv"
"strings"
"time"
)
func readVersion(stateDir string) (int, error) {
path := filepath.Join(stateDir, "version")
fileBytes, err := os.ReadFile(path)
if errors.Is(err, fs.ErrNotExist) {
if fileExists(filepath.Join(stateDir, "evidence")) {
return 0, nil
} else {
return -1, nil
}
} else if err != nil {
return -1, err
}
version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes)))
if err != nil {
return -1, fmt.Errorf("version file %q is malformed: %w", path, err)
}
return version, nil
}
func writeVersion(stateDir string) error {
return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666)
}
func migrateLogStateDirV1(dir string) error {
var sth ct.SignedTreeHead
var tree merkletree.CollapsedTree
sthPath := filepath.Join(dir, "sth.json")
sthData, err := os.ReadFile(sthPath)
if errors.Is(err, fs.ErrNotExist) {
return nil
} else if err != nil {
return err
}
treePath := filepath.Join(dir, "tree.json")
treeData, err := os.ReadFile(treePath)
if errors.Is(err, fs.ErrNotExist) {
return nil
} else if err != nil {
return err
}
if err := json.Unmarshal(sthData, &sth); err != nil {
return fmt.Errorf("error unmarshaling %s: %w", sthPath, err)
}
if err := json.Unmarshal(treeData, &tree); err != nil {
return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
}
stateFile := stateFile{
DownloadPosition: &tree,
VerifiedPosition: &tree,
VerifiedSTH: &sth,
LastSuccess: time.Now().UTC(),
}
if stateFile.store(filepath.Join(dir, "state.json")); err != nil {
return err
}
if err := os.Remove(sthPath); err != nil {
return err
}
if err := os.Remove(treePath); err != nil {
return err
}
return nil
}
func migrateStateDirV1(stateDir string) error {
if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) {
return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile)
}
if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil {
for _, logDir := range logDirs {
if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() {
continue
}
if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil {
return fmt.Errorf("error migrating log state: %w", err)
}
}
} else if !errors.Is(err, fs.ErrNotExist) {
return err
}
if err := writeVersion(stateDir); err != nil {
return err
}
if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
return nil
}
func prepareStateDir(stateDir string) error {
if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return err
}
if version, err := readVersion(stateDir); err != nil {
return err
} else if version == -1 {
if err := writeVersion(stateDir); err != nil {
return err
}
} else if version == 0 {
return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir)
} else if version == 1 {
if err := migrateStateDirV1(stateDir); err != nil {
return err
}
} else if version > 2 {
return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
}
for _, subdir := range []string{"certs", "logs"} {
if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
return err
}
}
return nil
}

47
monitor/statefile.go Normal file
View File

@ -0,0 +1,47 @@
// Copyright (C) 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"encoding/json"
"fmt"
"os"
"software.sslmate.com/src/certspotter/ct"
"software.sslmate.com/src/certspotter/merkletree"
"time"
)
type stateFile struct {
DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"`
LastSuccess time.Time `json:"last_success"`
}
func loadStateFile(filePath string) (*stateFile, error) {
fileBytes, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
file := new(stateFile)
if err := json.Unmarshal(fileBytes, file); err != nil {
return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
}
return file, nil
}
func (file *stateFile) store(filePath string) error {
fileBytes, err := json.Marshal(file)
if err != nil {
return err
}
fileBytes = append(fileBytes, '\n')
return writeFile(filePath, fileBytes, 0666)
}

96
monitor/sthdir.go Normal file
View File

@ -0,0 +1,96 @@
// Copyright (C) 2017, 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"golang.org/x/exp/slices"
"io/fs"
"os"
"path/filepath"
"software.sslmate.com/src/certspotter/ct"
"strconv"
"strings"
)
func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
entries, err := os.ReadDir(dirPath)
if errors.Is(err, fs.ErrNotExist) {
return []*ct.SignedTreeHead{}, nil
} else if err != nil {
return nil, err
}
sths := make([]*ct.SignedTreeHead, 0, len(entries))
for _, entry := range entries {
filename := entry.Name()
if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
continue
}
sth, err := readSTHFile(filepath.Join(dirPath, filename))
if err != nil {
return nil, err
}
sths = append(sths, sth)
}
slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) bool { return a.TreeSize < b.TreeSize })
return sths, nil
}
func readSTHFile(filePath string) (*ct.SignedTreeHead, error) {
fileBytes, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
sth := new(ct.SignedTreeHead)
if err := json.Unmarshal(fileBytes, sth); err != nil {
return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
}
return sth, nil
}
func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
filePath := filepath.Join(dirPath, sthFilename(sth))
if fileExists(filePath) {
return nil
}
fileBytes, err := json.Marshal(sth)
if err != nil {
return err
}
return writeFile(filePath, fileBytes, 0666)
}
func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
filePath := filepath.Join(dirPath, sthFilename(sth))
err := os.Remove(filePath)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
return nil
}
// generate a filename that uniquely identifies the STH (within the context of a particular log)
func sthFilename(sth *ct.SignedTreeHead) string {
hasher := sha256.New()
switch sth.Version {
case ct.V1:
binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
default:
panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version))
}
// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
}

135
monitor/watchlist.go Normal file
View File

@ -0,0 +1,135 @@
// Copyright (C) 2016, 2023 Opsmate, Inc.
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License, v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
// See the Mozilla Public License for details.
package monitor
import (
"bufio"
"fmt"
"golang.org/x/net/idna"
"io"
"software.sslmate.com/src/certspotter"
"strings"
)
type WatchItem struct {
domain []string
acceptSuffix bool
}
type WatchList []WatchItem
func ParseWatchItem(str string) (WatchItem, error) {
fields := strings.Fields(str)
if len(fields) == 0 {
return WatchItem{}, fmt.Errorf("empty domain")
}
domain := fields[0]
for _, field := range fields[1:] {
switch {
case strings.HasPrefix(field, "valid_at:"):
// Ignore for backwards compatibility
default:
return WatchItem{}, fmt.Errorf("unknown parameter %q", field)
}
}
if domain == "." {
// "." as in root zone -> matches everything
return WatchItem{
domain: []string{},
acceptSuffix: true,
}, nil
}
acceptSuffix := false
if strings.HasPrefix(domain, ".") {
acceptSuffix = true
domain = domain[1:]
}
asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, ".")))
if err != nil {
return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err)
}
return WatchItem{
domain: strings.Split(asciiDomain, "."),
acceptSuffix: acceptSuffix,
}, nil
}
func ReadWatchList(reader io.Reader) (WatchList, error) {
items := make(WatchList, 0, 50)
scanner := bufio.NewScanner(reader)
lineNo := 0
for scanner.Scan() {
line := scanner.Text()
lineNo++
if line == "" || strings.HasPrefix(line, "#") {
continue
}
item, err := ParseWatchItem(line)
if err != nil {
return nil, fmt.Errorf("%w on line %d", err, lineNo)
}
items = append(items, item)
}
return items, scanner.Err()
}
func (item WatchItem) String() string {
if item.acceptSuffix {
return "." + strings.Join(item.domain, ".")
} else {
return strings.Join(item.domain, ".")
}
}
func (item WatchItem) matchesDNSName(dnsName []string) bool {
watchDomain := item.domain
for len(dnsName) > 0 && len(watchDomain) > 0 {
certLabel := dnsName[len(dnsName)-1]
watchLabel := watchDomain[len(watchDomain)-1]
if !dnsLabelMatches(certLabel, watchLabel) {
return false
}
dnsName = dnsName[:len(dnsName)-1]
watchDomain = watchDomain[:len(watchDomain)-1]
}
return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0)
}
func dnsLabelMatches(certLabel string, watchLabel string) bool {
// For fail-safe behavior, if a label was unparsable, it matches everything.
// Similarly, redacted labels match everything, since the label _might_ be
// for a name we're interested in.
return certLabel == "*" ||
certLabel == "?" ||
certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
certspotter.MatchesWildcard(watchLabel, certLabel)
}
func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) {
dnsNames := make([][]string, len(identifiers.DNSNames))
for i, dnsName := range identifiers.DNSNames {
dnsNames[i] = strings.Split(dnsName, ".")
}
for _, item := range list {
for _, dnsName := range dnsNames {
if item.matchesDNSName(dnsName) {
return true, item
}
}
}
return false, WatchItem{}
}