Convert to a daemon and make many other improvements
Specifically, certspotter no longer terminates unless it receives SIGTERM or SIGINT or there is a serious error. Although using cron made sense in the early days of Certificate Transparency, certspotter now needs to run continuously to reliably keep up with the high growth rate of contemporary CT logs, and to gracefully handle the many transient errors that can arise when monitoring CT. Closes: #63 Closes: #37 Closes: #32 (presumably by eliminating $DNS_NAMES and $IP_ADDRESSES) Closes: #21 (with $WATCH_ITEM) Closes: #25
This commit is contained in:
Andrew Ayer
parent
e3835dea53
commit
209cdb181b
|
|
@ -1,4 +1,4 @@
|
|||
// Copyright (C) 2016 Opsmate, Inc.
|
||||
// Copyright (C) 2016, 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
|
|
@ -10,224 +10,169 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
insecurerand "math/rand"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/idna"
|
||||
|
||||
"software.sslmate.com/src/certspotter"
|
||||
"software.sslmate.com/src/certspotter/cmd"
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"software.sslmate.com/src/certspotter/monitor"
|
||||
)
|
||||
|
||||
var programName = os.Args[0]
|
||||
|
||||
const defaultLogList = "https://loglist.certspotter.org/monitor.json"
|
||||
|
||||
func certspotterVersion() string {
|
||||
info, ok := debug.ReadBuildInfo()
|
||||
if !ok {
|
||||
return "unknown"
|
||||
}
|
||||
if strings.HasPrefix(info.Main.Version, "v") {
|
||||
return info.Main.Version
|
||||
}
|
||||
var vcs, vcsRevision, vcsModified string
|
||||
for _, s := range info.Settings {
|
||||
switch s.Key {
|
||||
case "vcs":
|
||||
vcs = s.Value
|
||||
case "vcs.revision":
|
||||
vcsRevision = s.Value
|
||||
case "vcs.modified":
|
||||
vcsModified = s.Value
|
||||
}
|
||||
}
|
||||
if vcs == "git" && vcsRevision != "" && vcsModified == "true" {
|
||||
return vcsRevision + "+"
|
||||
} else if vcs == "git" && vcsRevision != "" {
|
||||
return vcsRevision
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func homedir() string {
|
||||
homedir, err := os.UserHomeDir()
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("unable to determine home directory: %w", err))
|
||||
}
|
||||
return homedir
|
||||
}
|
||||
func defaultStateDir() string {
|
||||
if envVar := os.Getenv("CERTSPOTTER_STATE_DIR"); envVar != "" {
|
||||
return envVar
|
||||
} else {
|
||||
return cmd.DefaultStateDir("certspotter")
|
||||
return filepath.Join(homedir(), ".certspotter")
|
||||
}
|
||||
}
|
||||
func defaultConfigDir() string {
|
||||
if envVar := os.Getenv("CERTSPOTTER_CONFIG_DIR"); envVar != "" {
|
||||
return envVar
|
||||
} else {
|
||||
return cmd.DefaultConfigDir("certspotter")
|
||||
return filepath.Join(homedir(), ".certspotter")
|
||||
}
|
||||
}
|
||||
|
||||
func trimTrailingDots(value string) string {
|
||||
length := len(value)
|
||||
for length > 0 && value[length-1] == '.' {
|
||||
length--
|
||||
}
|
||||
return value[0:length]
|
||||
}
|
||||
|
||||
var stateDir = flag.String("state_dir", defaultStateDir(), "Directory for storing state")
|
||||
var watchlistFilename = flag.String("watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing identifiers to watch (- for stdin)")
|
||||
|
||||
type watchlistItem struct {
|
||||
Domain []string
|
||||
AcceptSuffix bool
|
||||
ValidAt *time.Time // optional
|
||||
}
|
||||
|
||||
var watchlist []watchlistItem
|
||||
|
||||
func parseWatchlistItem(str string) (watchlistItem, error) {
|
||||
fields := strings.Fields(str)
|
||||
if len(fields) == 0 {
|
||||
return watchlistItem{}, fmt.Errorf("Empty domain")
|
||||
}
|
||||
domain := fields[0]
|
||||
var validAt *time.Time = nil
|
||||
|
||||
// parse options
|
||||
for i := 1; i < len(fields); i++ {
|
||||
chunks := strings.SplitN(fields[i], ":", 2)
|
||||
if len(chunks) != 2 {
|
||||
return watchlistItem{}, fmt.Errorf("Missing Value `%s'", fields[i])
|
||||
}
|
||||
switch chunks[0] {
|
||||
case "valid_at":
|
||||
validAtTime, err := time.Parse("2006-01-02", chunks[1])
|
||||
func readWatchListFile(filename string) (monitor.WatchList, error) {
|
||||
file, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return watchlistItem{}, fmt.Errorf("Invalid Date `%s': %s", chunks[1], err)
|
||||
var pathErr *fs.PathError
|
||||
if errors.As(err, &pathErr) {
|
||||
err = pathErr.Err
|
||||
}
|
||||
validAt = &validAtTime
|
||||
default:
|
||||
return watchlistItem{}, fmt.Errorf("Unknown Option `%s'", fields[i])
|
||||
}
|
||||
}
|
||||
|
||||
// parse domain
|
||||
// "." as in root zone (matches everything)
|
||||
if domain == "." {
|
||||
return watchlistItem{
|
||||
Domain: []string{},
|
||||
AcceptSuffix: true,
|
||||
ValidAt: validAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
acceptSuffix := false
|
||||
if strings.HasPrefix(domain, ".") {
|
||||
acceptSuffix = true
|
||||
domain = domain[1:]
|
||||
}
|
||||
|
||||
asciiDomain, err := idna.ToASCII(strings.ToLower(trimTrailingDots(domain)))
|
||||
if err != nil {
|
||||
return watchlistItem{}, fmt.Errorf("Invalid domain `%s': %s", domain, err)
|
||||
}
|
||||
return watchlistItem{
|
||||
Domain: strings.Split(asciiDomain, "."),
|
||||
AcceptSuffix: acceptSuffix,
|
||||
ValidAt: validAt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func readWatchlist(reader io.Reader) ([]watchlistItem, error) {
|
||||
items := []watchlistItem{}
|
||||
scanner := bufio.NewScanner(reader)
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
item, err := parseWatchlistItem(line)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, scanner.Err()
|
||||
defer file.Close()
|
||||
return monitor.ReadWatchList(file)
|
||||
}
|
||||
|
||||
func dnsLabelMatches(certLabel string, watchLabel string) bool {
|
||||
// For fail-safe behavior, if a label was unparsable, it matches everything.
|
||||
// Similarly, redacted labels match everything, since the label _might_ be
|
||||
// for a name we're interested in.
|
||||
|
||||
return certLabel == "*" ||
|
||||
certLabel == "?" ||
|
||||
certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
|
||||
certspotter.MatchesWildcard(watchLabel, certLabel)
|
||||
}
|
||||
|
||||
func dnsNameMatches(dnsName []string, watchDomain []string, acceptSuffix bool) bool {
|
||||
for len(dnsName) > 0 && len(watchDomain) > 0 {
|
||||
certLabel := dnsName[len(dnsName)-1]
|
||||
watchLabel := watchDomain[len(watchDomain)-1]
|
||||
|
||||
if !dnsLabelMatches(certLabel, watchLabel) {
|
||||
return false
|
||||
}
|
||||
|
||||
dnsName = dnsName[:len(dnsName)-1]
|
||||
watchDomain = watchDomain[:len(watchDomain)-1]
|
||||
}
|
||||
|
||||
return len(watchDomain) == 0 && (acceptSuffix || len(dnsName) == 0)
|
||||
}
|
||||
|
||||
func anyDnsNameIsWatched(info *certspotter.EntryInfo) bool {
|
||||
dnsNames := info.Identifiers.DNSNames
|
||||
matched := false
|
||||
for _, dnsName := range dnsNames {
|
||||
labels := strings.Split(dnsName, ".")
|
||||
for _, item := range watchlist {
|
||||
if dnsNameMatches(labels, item.Domain, item.AcceptSuffix) {
|
||||
if item.ValidAt != nil {
|
||||
// BygoneSSL Check
|
||||
// was the SSL certificate issued before the domain was registered
|
||||
// and valid after
|
||||
if item.ValidAt.Before(*info.CertInfo.NotAfter()) &&
|
||||
item.ValidAt.After(*info.CertInfo.NotBefore()) {
|
||||
info.Bygone = true
|
||||
return true
|
||||
}
|
||||
}
|
||||
// keep iterating in case another domain watched matches valid_at
|
||||
matched = true
|
||||
}
|
||||
}
|
||||
}
|
||||
return matched
|
||||
}
|
||||
|
||||
func processEntry(scanner *certspotter.Scanner, entry *ct.LogEntry) {
|
||||
info := certspotter.EntryInfo{
|
||||
LogUri: scanner.LogUri,
|
||||
Entry: entry,
|
||||
IsPrecert: certspotter.IsPrecert(entry),
|
||||
FullChain: certspotter.GetFullChain(entry),
|
||||
}
|
||||
|
||||
info.CertInfo, info.ParseError = certspotter.MakeCertInfoFromLogEntry(entry)
|
||||
|
||||
if info.CertInfo != nil {
|
||||
info.Identifiers, info.IdentifiersParseError = info.CertInfo.ParseIdentifiers()
|
||||
}
|
||||
|
||||
// Fail safe behavior: if info.Identifiers is nil (which is caused by a
|
||||
// parse error), report the certificate because we can't say for sure it
|
||||
// doesn't match a domain we care about. We try very hard to make sure
|
||||
// parsing identifiers always succeeds, so false alarms should be rare.
|
||||
if info.Identifiers == nil || anyDnsNameIsWatched(&info) {
|
||||
cmd.LogEntry(&info)
|
||||
func appendFunc(slice *[]string) func(string) error {
|
||||
return func(value string) error {
|
||||
*slice = append(*slice, value)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
cmd.ParseFlags()
|
||||
insecurerand.Seed(time.Now().UnixNano()) // TODO: remove after upgrading to Go 1.20
|
||||
|
||||
if *watchlistFilename == "-" {
|
||||
var err error
|
||||
watchlist, err = readWatchlist(os.Stdin)
|
||||
// TODO-3: set loglist.UserAgent
|
||||
|
||||
var flags struct {
|
||||
batchSize int // TODO-4: respect this option
|
||||
email []string
|
||||
logs string
|
||||
noSave bool
|
||||
script string
|
||||
startAtEnd bool
|
||||
stateDir string
|
||||
stdout bool
|
||||
verbose bool
|
||||
version bool
|
||||
watchlist string
|
||||
}
|
||||
flag.IntVar(&flags.batchSize, "batch_size", 1000, "Max number of entries to request per call to get-entries (advanced)")
|
||||
flag.Func("email", "Email address to contact when matching certificate is discovered (repeatable)", appendFunc(&flags.email))
|
||||
flag.StringVar(&flags.logs, "logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
|
||||
flag.BoolVar(&flags.noSave, "no_save", false, "Do not save a copy of matching certificates in state directory")
|
||||
flag.StringVar(&flags.script, "script", "", "Program to execute when a matching certificate is discovered")
|
||||
flag.BoolVar(&flags.startAtEnd, "start_at_end", false, "Start monitoring logs from the end rather than the beginning (saves considerable bandwidth)")
|
||||
flag.StringVar(&flags.stateDir, "state_dir", defaultStateDir(), "Directory for storing log position and discovered certificates")
|
||||
flag.BoolVar(&flags.stdout, "stdout", false, "Write matching certificates to stdout")
|
||||
flag.BoolVar(&flags.verbose, "verbose", false, "Be verbose")
|
||||
flag.BoolVar(&flags.version, "version", false, "Print version and exit")
|
||||
flag.StringVar(&flags.watchlist, "watchlist", filepath.Join(defaultConfigDir(), "watchlist"), "File containing domain names to watch")
|
||||
flag.Parse()
|
||||
|
||||
if flags.version {
|
||||
fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
if len(flags.email) == 0 && len(flags.script) == 0 && flags.stdout == false {
|
||||
fmt.Fprintf(os.Stderr, "%s: at least one of -email, -script, or -stdout must be specified (see -help for details)\n", programName)
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
config := &monitor.Config{
|
||||
LogListSource: flags.logs,
|
||||
StateDir: flags.stateDir,
|
||||
SaveCerts: !flags.noSave,
|
||||
StartAtEnd: flags.startAtEnd,
|
||||
Verbose: flags.verbose,
|
||||
Script: flags.script,
|
||||
Email: flags.email,
|
||||
Stdout: flags.stdout,
|
||||
}
|
||||
|
||||
if flags.watchlist == "-" {
|
||||
watchlist, err := monitor.ReadWatchList(os.Stdin)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: (stdin): %s\n", os.Args[0], err)
|
||||
fmt.Fprintf(os.Stderr, "%s: error reading watchlist from standard in: %s\n", programName, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
config.WatchList = watchlist
|
||||
} else {
|
||||
file, err := os.Open(*watchlistFilename)
|
||||
watchlist, err := readWatchListFile(flags.watchlist)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
watchlist, err = readWatchlist(file)
|
||||
file.Close()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %s: %s\n", os.Args[0], *watchlistFilename, err)
|
||||
fmt.Fprintf(os.Stderr, "%s: error reading watchlist from %q: %s\n", programName, flags.watchlist, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
config.WatchList = watchlist
|
||||
}
|
||||
|
||||
os.Exit(cmd.Main(*stateDir, processEntry))
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
if err := monitor.Run(ctx, config); err != nil && !errors.Is(err, context.Canceled) {
|
||||
fmt.Fprintf(os.Stderr, "%s: %s\n", programName, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
8
go.mod
8
go.mod
|
|
@ -1,8 +1,10 @@
|
|||
module software.sslmate.com/src/certspotter
|
||||
|
||||
go 1.17
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 // indirect
|
||||
golang.org/x/text v0.3.7 // indirect
|
||||
golang.org/x/exp v0.0.0-20230118134722-a68e582fa157 // indirect
|
||||
golang.org/x/net v0.5.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/text v0.6.0 // indirect
|
||||
)
|
||||
|
|
|
|||
8
go.sum
8
go.sum
|
|
@ -1,4 +1,12 @@
|
|||
golang.org/x/exp v0.0.0-20230118134722-a68e582fa157 h1:fiNkyhJPUvxbRPbCqY/D9qdjmPzfHcpK3P4bM4gioSY=
|
||||
golang.org/x/exp v0.0.0-20230118134722-a68e582fa157/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4 h1:HVyaeDAYux4pnY+D/SiwmLOR36ewZ4iGQIIrtnuCjFA=
|
||||
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
|
||||
golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
|
||||
golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
|
||||
golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
type Config struct {
|
||||
LogListSource string
|
||||
StateDir string
|
||||
StartAtEnd bool
|
||||
WatchList WatchList
|
||||
Verbose bool
|
||||
SaveCerts bool
|
||||
Script string
|
||||
Email []string
|
||||
Stdout bool
|
||||
}
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"log"
|
||||
insecurerand "math/rand"
|
||||
"software.sslmate.com/src/certspotter/loglist"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
reloadLogListIntervalMin = 3 * time.Second // TODO-3: use 30 * time.Minute
|
||||
reloadLogListIntervalMax = 9 * time.Second // TODO-3: use 90 * time.Minute
|
||||
healthCheckInterval = 24 * time.Hour
|
||||
)
|
||||
|
||||
func randomDuration(min, max time.Duration) time.Duration {
|
||||
return min + time.Duration(insecurerand.Int63n(int64(max-min+1)))
|
||||
}
|
||||
|
||||
func reloadLogListInterval() time.Duration {
|
||||
return randomDuration(reloadLogListIntervalMin, reloadLogListIntervalMax)
|
||||
}
|
||||
|
||||
type task struct {
|
||||
stop context.CancelFunc
|
||||
}
|
||||
|
||||
type daemon struct {
|
||||
config *Config
|
||||
taskgroup *errgroup.Group
|
||||
tasks map[LogID]task
|
||||
logsLoadedAt time.Time
|
||||
}
|
||||
|
||||
func (daemon *daemon) healthCheck(ctx context.Context) error { // TODO-2
|
||||
return nil
|
||||
}
|
||||
|
||||
func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
daemon.taskgroup.Go(func() error {
|
||||
defer cancel()
|
||||
err := monitorLogContinously(ctx, daemon.config, ctlog)
|
||||
if daemon.config.Verbose {
|
||||
log.Printf("task for log %s stopped with error %s", ctlog.URL, err)
|
||||
}
|
||||
if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
|
||||
return nil
|
||||
} else {
|
||||
return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err)
|
||||
}
|
||||
})
|
||||
return task{stop: cancel}
|
||||
}
|
||||
|
||||
func (daemon *daemon) loadLogList(ctx context.Context) error {
|
||||
loglist, err := getLogList(ctx, daemon.config.LogListSource)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if daemon.config.Verbose {
|
||||
log.Printf("fetched %d logs from %q", len(loglist), daemon.config.LogListSource)
|
||||
}
|
||||
|
||||
for logID, task := range daemon.tasks {
|
||||
if _, exists := loglist[logID]; exists {
|
||||
continue
|
||||
}
|
||||
if daemon.config.Verbose {
|
||||
log.Printf("stopping task for log %s", logID.Base64String())
|
||||
}
|
||||
task.stop()
|
||||
delete(daemon.tasks, logID)
|
||||
}
|
||||
for logID, ctlog := range loglist {
|
||||
if _, isRunning := daemon.tasks[logID]; isRunning {
|
||||
continue
|
||||
}
|
||||
if daemon.config.Verbose {
|
||||
log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL)
|
||||
}
|
||||
daemon.tasks[logID] = daemon.startTask(ctx, ctlog)
|
||||
}
|
||||
daemon.logsLoadedAt = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (daemon *daemon) run(ctx context.Context) error {
|
||||
if err := prepareStateDir(daemon.config.StateDir); err != nil {
|
||||
return fmt.Errorf("error preparing state directory: %w", err)
|
||||
}
|
||||
|
||||
if err := daemon.loadLogList(ctx); err != nil {
|
||||
return fmt.Errorf("error loading log list: %w", err)
|
||||
}
|
||||
|
||||
reloadLogListTicker := time.NewTicker(reloadLogListInterval())
|
||||
defer reloadLogListTicker.Stop()
|
||||
|
||||
healthCheckTicker := time.NewTicker(healthCheckInterval)
|
||||
defer healthCheckTicker.Stop()
|
||||
|
||||
for ctx.Err() == nil {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-reloadLogListTicker.C:
|
||||
if err := daemon.loadLogList(ctx); err != nil {
|
||||
recordError(fmt.Errorf("error reloading log list (will try again later): %w", err))
|
||||
}
|
||||
reloadLogListTicker.Reset(reloadLogListInterval())
|
||||
case <-healthCheckTicker.C:
|
||||
if err := daemon.healthCheck(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func Run(ctx context.Context, config *Config) error {
|
||||
group, ctx := errgroup.WithContext(ctx)
|
||||
daemon := &daemon{
|
||||
config: config,
|
||||
taskgroup: group,
|
||||
tasks: make(map[LogID]task),
|
||||
}
|
||||
group.Go(func() error { return daemon.run(ctx) })
|
||||
return group.Wait()
|
||||
}
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"software.sslmate.com/src/certspotter"
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
)
|
||||
|
||||
type discoveredCert struct {
|
||||
WatchItem WatchItem
|
||||
LogEntry *logEntry
|
||||
Info *certspotter.CertInfo
|
||||
Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate
|
||||
LeafSHA256 [32]byte // computed over Chain[0]
|
||||
Identifiers *certspotter.Identifiers
|
||||
CertPath string // empty if not saved on the filesystem
|
||||
JSONPath string // empty if not saved on the filesystem
|
||||
TextPath string // empty if not saved on the filesystem
|
||||
}
|
||||
|
||||
func (cert *discoveredCert) pemChain() []byte {
|
||||
var buffer bytes.Buffer
|
||||
for _, certBytes := range cert.Chain {
|
||||
if err := pem.Encode(&buffer, &pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: certBytes,
|
||||
}); err != nil {
|
||||
panic(fmt.Errorf("encoding certificate as PEM failed unexpectedly: %w", err))
|
||||
}
|
||||
}
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func (cert *discoveredCert) json() []byte {
|
||||
pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey())
|
||||
|
||||
object := map[string]any{
|
||||
"cert_sha256": hex.EncodeToString(cert.LeafSHA256[:]),
|
||||
"pubkey_sha256": hex.EncodeToString(pubkeySha256[:]),
|
||||
"issuer_der": cert.Info.TBS.Issuer.FullBytes,
|
||||
"subject_der": cert.Info.TBS.Subject.FullBytes,
|
||||
"dns_names": cert.Identifiers.DNSNames,
|
||||
"ip_addresses": cert.Identifiers.IPAddrs,
|
||||
}
|
||||
|
||||
if cert.Info.ValidityParseError == nil {
|
||||
object["not_before"] = cert.Info.Validity.NotBefore
|
||||
object["not_after"] = cert.Info.Validity.NotAfter
|
||||
} else {
|
||||
object["not_before"] = nil
|
||||
object["not_after"] = nil
|
||||
}
|
||||
|
||||
if cert.Info.SerialNumberParseError == nil {
|
||||
object["serial_number"] = fmt.Sprintf("%x", cert.Info.SerialNumber)
|
||||
} else {
|
||||
object["serial_number"] = nil
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(object)
|
||||
if err != nil {
|
||||
panic(fmt.Errorf("encoding certificate as JSON failed unexpectedly: %w", err))
|
||||
}
|
||||
return jsonBytes
|
||||
}
|
||||
|
||||
func (cert *discoveredCert) save() error {
|
||||
if err := writeFile(cert.CertPath, cert.pemChain(), 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeFile(cert.JSONPath, cert.json(), 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := writeFile(cert.TextPath, []byte(cert.Text()), 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cert *discoveredCert) Environ() []string {
|
||||
pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey())
|
||||
|
||||
env := []string{
|
||||
"EVENT=discovered_cert",
|
||||
"SUMMARY=certificate discovered for " + cert.WatchItem.String(),
|
||||
"CERT_PARSEABLE=yes", // backwards compat; not documented (TODO-3: consider removing)
|
||||
"LOG_URI=" + cert.LogEntry.Log.URL,
|
||||
"ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index),
|
||||
"WATCH_ITEM=" + cert.WatchItem.String(),
|
||||
"CERT_SHA256=" + hex.EncodeToString(cert.LeafSHA256[:]),
|
||||
"FINGERPRINT=" + hex.EncodeToString(cert.LeafSHA256[:]), // backwards compat; not documented (TODO-3: consider removing)
|
||||
"PUBKEY_SHA256=" + hex.EncodeToString(pubkeySha256[:]),
|
||||
"PUBKEY_HASH=" + hex.EncodeToString(pubkeySha256[:]), // backwards compat; not documented (TODO-3: consider removing)
|
||||
"CERT_FILENAME=" + cert.CertPath,
|
||||
"JSON_FILENAME=" + cert.JSONPath,
|
||||
"TEXT_FILENAME=" + cert.TextPath,
|
||||
}
|
||||
|
||||
if cert.Info.ValidityParseError == nil {
|
||||
env = append(env, "NOT_BEFORE="+cert.Info.Validity.NotBefore.String())
|
||||
env = append(env, "NOT_BEFORE_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotBefore.Unix()))
|
||||
env = append(env, "NOT_BEFORE_RFC3339="+cert.Info.Validity.NotBefore.Format(time.RFC3339))
|
||||
env = append(env, "NOT_AFTER="+cert.Info.Validity.NotAfter.String())
|
||||
env = append(env, "NOT_AFTER_UNIXTIME="+fmt.Sprint(cert.Info.Validity.NotAfter.Unix()))
|
||||
env = append(env, "NOT_AFTER_RFC3339="+cert.Info.Validity.NotAfter.Format(time.RFC3339))
|
||||
} else {
|
||||
env = append(env, "VALIDITY_PARSE_ERROR="+cert.Info.ValidityParseError.Error())
|
||||
}
|
||||
|
||||
if cert.Info.SubjectParseError == nil {
|
||||
env = append(env, "SUBJECT_DN="+cert.Info.Subject.String())
|
||||
} else {
|
||||
env = append(env, "SUBJECT_PARSE_ERROR="+cert.Info.SubjectParseError.Error())
|
||||
}
|
||||
|
||||
if cert.Info.IssuerParseError == nil {
|
||||
env = append(env, "ISSUER_DN="+cert.Info.Issuer.String())
|
||||
} else {
|
||||
env = append(env, "ISSUER_PARSE_ERROR="+cert.Info.IssuerParseError.Error())
|
||||
}
|
||||
|
||||
// TODO-3: consider removing $SERIAL due to misuse potential, or renaming to $SERIAL_NUMBER
|
||||
if cert.Info.SerialNumberParseError == nil {
|
||||
env = append(env, "SERIAL="+fmt.Sprintf("%x", cert.Info.SerialNumber))
|
||||
} else {
|
||||
env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error())
|
||||
}
|
||||
|
||||
return env
|
||||
}
|
||||
|
||||
func (cert *discoveredCert) Text() string {
|
||||
// TODO-3: improve the output: include WatchItem, indicate hash algorithm. look at sslmate email for inspiration
|
||||
|
||||
text := new(strings.Builder)
|
||||
writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
|
||||
|
||||
pubkeySha256 := sha256.Sum256(cert.Info.TBS.GetRawPublicKey())
|
||||
|
||||
fmt.Fprintf(text, "%x:\n", cert.LeafSHA256)
|
||||
for _, dnsName := range cert.Identifiers.DNSNames {
|
||||
writeField("DNS Name", dnsName)
|
||||
}
|
||||
for _, ipaddr := range cert.Identifiers.IPAddrs {
|
||||
writeField("IP Address", ipaddr)
|
||||
}
|
||||
writeField("Pubkey", hex.EncodeToString(pubkeySha256[:]))
|
||||
if cert.Info.IssuerParseError == nil {
|
||||
writeField("Issuer", cert.Info.Issuer)
|
||||
} else {
|
||||
writeField("Issuer", fmt.Sprintf("[unable to parse: %s]", cert.Info.IssuerParseError))
|
||||
}
|
||||
if cert.Info.ValidityParseError == nil {
|
||||
writeField("Not Before", cert.Info.Validity.NotBefore)
|
||||
writeField("Not After", cert.Info.Validity.NotAfter)
|
||||
} else {
|
||||
writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
|
||||
writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError))
|
||||
}
|
||||
writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL)) // TODO-3: include entry type here?
|
||||
writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.LeafSHA256[:]))
|
||||
if cert.CertPath != "" {
|
||||
writeField("Filename", cert.CertPath)
|
||||
}
|
||||
|
||||
return text.String()
|
||||
}
|
||||
|
||||
func (cert *discoveredCert) EmailSubject() string {
|
||||
return fmt.Sprintf("[certspotter] Certificate Discovered for %s", cert.WatchItem)
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"log"
|
||||
)
|
||||
|
||||
func recordError(err error) {
|
||||
log.Print(err)
|
||||
}
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
// Copyright (C) 2017, 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
func randomFileSuffix() string {
|
||||
var randomBytes [12]byte
|
||||
if _, err := rand.Read(randomBytes[:]); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hex.EncodeToString(randomBytes[:])
|
||||
}
|
||||
|
||||
func writeFile(filename string, data []byte, perm os.FileMode) error {
|
||||
tempname := filename + ".tmp." + randomFileSuffix()
|
||||
if err := os.WriteFile(tempname, data, perm); err != nil {
|
||||
return fmt.Errorf("error writing %s: %w", filename, err)
|
||||
}
|
||||
if err := os.Rename(tempname, filename); err != nil {
|
||||
os.Remove(tempname)
|
||||
return fmt.Errorf("error writing %s: %w", filename, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileExists(filename string) bool {
|
||||
_, err := os.Lstat(filename)
|
||||
return err == nil
|
||||
}
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"software.sslmate.com/src/certspotter/loglist"
|
||||
)
|
||||
|
||||
type staleSTHEvent struct {
|
||||
Log *loglist.Log
|
||||
LastSuccess time.Time
|
||||
LatestSTH *ct.SignedTreeHead // may be nil
|
||||
}
|
||||
type backlogEvent struct {
|
||||
Log *loglist.Log
|
||||
LatestSTH *ct.SignedTreeHead
|
||||
Backlog uint64
|
||||
Position uint64
|
||||
}
|
||||
type staleLogListEvent struct {
|
||||
Source string
|
||||
LastSuccess time.Time
|
||||
LastError string
|
||||
LastErrorTime time.Time
|
||||
}
|
||||
|
||||
func (e *staleSTHEvent) Environ() []string {
|
||||
return []string{
|
||||
"EVENT=error",
|
||||
"SUMMARY=" + fmt.Sprintf("unable to contact %s since %s", e.Log.URL, e.LastSuccess),
|
||||
}
|
||||
}
|
||||
func (e *backlogEvent) Environ() []string {
|
||||
return []string{
|
||||
"EVENT=error",
|
||||
"SUMMARY=" + fmt.Sprintf("backlog of size %d from %s", e.Backlog, e.Log.URL),
|
||||
}
|
||||
}
|
||||
func (e *staleLogListEvent) Environ() []string {
|
||||
return []string{
|
||||
"EVENT=error",
|
||||
"SUMMARY=" + fmt.Sprintf("unable to retrieve log list since %s: %s", e.LastSuccess, e.LastError),
|
||||
}
|
||||
}
|
||||
|
||||
func (e *staleSTHEvent) EmailSubject() string {
|
||||
return fmt.Sprintf("[certspotter] Unable to contact %s since %s", e.Log.URL, e.LastSuccess)
|
||||
}
|
||||
func (e *backlogEvent) EmailSubject() string {
|
||||
return fmt.Sprintf("[certspotter] Backlog of size %d from %s", e.Backlog, e.Log.URL)
|
||||
}
|
||||
func (e *staleLogListEvent) EmailSubject() string {
|
||||
return fmt.Sprintf("[certspotter] Unable to retrieve log list since %s", e.LastSuccess)
|
||||
}
|
||||
|
||||
func (e *staleSTHEvent) Text() string {
|
||||
text := new(strings.Builder)
|
||||
fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess)
|
||||
fmt.Fprintf(text, "\n")
|
||||
fmt.Fprintf(text, "For details, see certspotter's stderr output.\n")
|
||||
fmt.Fprintf(text, "\n")
|
||||
if e.LatestSTH != nil {
|
||||
fmt.Fprintf(text, "Latest known log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.Timestamp)
|
||||
} else {
|
||||
fmt.Fprintf(text, "Latest known log size = none\n")
|
||||
}
|
||||
return text.String()
|
||||
}
|
||||
func (e *backlogEvent) Text() string {
|
||||
text := new(strings.Builder)
|
||||
fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL)
|
||||
fmt.Fprintf(text, "\n")
|
||||
fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n")
|
||||
fmt.Fprintf(text, "\n")
|
||||
fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.Timestamp)
|
||||
fmt.Fprintf(text, "Current position = %d\n", e.Position)
|
||||
fmt.Fprintf(text, " Backlog = %d\n", e.Backlog)
|
||||
return text.String()
|
||||
}
|
||||
func (e *staleLogListEvent) Text() string {
|
||||
text := new(strings.Builder)
|
||||
fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
|
||||
fmt.Fprintf(text, "\n")
|
||||
fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError)
|
||||
fmt.Fprintf(text, "\n")
|
||||
fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n")
|
||||
return text.String()
|
||||
}
|
||||
|
||||
// TODO-3: make the errors more actionable
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"software.sslmate.com/src/certspotter/loglist"
|
||||
)
|
||||
|
||||
type LogID = ct.SHA256Hash
|
||||
|
||||
func getLogList(ctx context.Context, source string) (map[LogID]*loglist.Log, error) {
|
||||
// TODO-4: pass context to loglist.Load
|
||||
// TODO-3: If-Modified-Since / If-None-Match support
|
||||
list, err := loglist.Load(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
logs := make(map[LogID]*loglist.Log)
|
||||
for operatorIndex := range list.Operators {
|
||||
for logIndex := range list.Operators[operatorIndex].Logs {
|
||||
log := &list.Operators[operatorIndex].Logs[logIndex]
|
||||
if _, exists := logs[log.LogID]; exists {
|
||||
return nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String())
|
||||
}
|
||||
logs[log.LogID] = log
|
||||
}
|
||||
}
|
||||
return logs, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type malformedLogEntry struct {
|
||||
Entry *logEntry
|
||||
Error string
|
||||
}
|
||||
|
||||
func (malformed *malformedLogEntry) Environ() []string {
|
||||
return []string{
|
||||
"EVENT=discovered_cert",
|
||||
"SUMMARY=" + fmt.Sprintf("unable to parse entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL),
|
||||
"LOG_URI=" + malformed.Entry.Log.URL,
|
||||
"ENTRY_INDEX=" + fmt.Sprint(malformed.Entry.Index),
|
||||
"LEAF_HASH=" + malformed.Entry.LeafHash.Base64String(),
|
||||
"PARSE_ERROR=" + malformed.Error,
|
||||
"CERT_PARSEABLE=no", // backwards compat; not documented (TODO-3: consider removing)
|
||||
}
|
||||
}
|
||||
|
||||
func (malformed *malformedLogEntry) Text() string {
|
||||
text := new(strings.Builder)
|
||||
writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) }
|
||||
|
||||
fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n")
|
||||
writeField("Log Entry", fmt.Sprintf("%d @ %s", malformed.Entry.Index, malformed.Entry.Log.URL))
|
||||
writeField("Leaf Hash", malformed.Entry.LeafHash.Base64String())
|
||||
writeField("Error", malformed.Error)
|
||||
|
||||
return text.String()
|
||||
}
|
||||
|
||||
func (malformed *malformedLogEntry) EmailSubject() string {
|
||||
return fmt.Sprintf("[certspotter] Unable to Parse Entry %d in %s", malformed.Entry.Index, malformed.Entry.Log.URL)
|
||||
}
|
||||
|
|
@ -0,0 +1,294 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"software.sslmate.com/src/certspotter/ct/client"
|
||||
"software.sslmate.com/src/certspotter/loglist"
|
||||
"software.sslmate.com/src/certspotter/merkletree"
|
||||
)
|
||||
|
||||
const (
|
||||
maxGetEntriesSize = 1000
|
||||
//monitorLogInterval = 5 * time.Minute // TODO-3
|
||||
monitorLogInterval = 15 * time.Second
|
||||
)
|
||||
|
||||
func isFatalLogError(err error) bool {
|
||||
return errors.Is(err, context.Canceled)
|
||||
}
|
||||
|
||||
func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) {
|
||||
logKey, err := x509.ParsePKIXPublicKey(ctlog.Key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error parsing log key: %w", err)
|
||||
}
|
||||
verifier, err := ct.NewSignatureVerifier(logKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error with log key: %w", err)
|
||||
}
|
||||
return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil
|
||||
}
|
||||
|
||||
func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error {
|
||||
logClient, err := newLogClient(ctlog)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ticker := time.NewTicker(monitorLogInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for ctx.Err() == nil {
|
||||
if err := monitorLog(ctx, config, ctlog, logClient); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func monitorLog(ctx context.Context, config *Config, ctlog *loglist.Log, logClient *client.LogClient) (returnedErr error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
var (
|
||||
stateDirPath = filepath.Join(config.StateDir, "logs", ctlog.LogID.Base64URLString())
|
||||
stateFilePath = filepath.Join(stateDirPath, "state.json")
|
||||
sthsDirPath = filepath.Join(stateDirPath, "unverified_sths")
|
||||
)
|
||||
for _, dirPath := range []string{stateDirPath, sthsDirPath} {
|
||||
if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("error creating state directory: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
latestSTH, err := logClient.GetSTH(ctx)
|
||||
if isFatalLogError(err) {
|
||||
return err
|
||||
} else if err != nil {
|
||||
recordError(fmt.Errorf("error fetching latest STH for %s: %w", ctlog.URL, err))
|
||||
return nil
|
||||
}
|
||||
latestSTH.LogID = ctlog.LogID
|
||||
if err := storeSTHInDir(sthsDirPath, latestSTH); err != nil {
|
||||
return fmt.Errorf("error storing latest STH: %w", err)
|
||||
}
|
||||
|
||||
state, err := loadStateFile(stateFilePath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if config.StartAtEnd {
|
||||
tree, err := reconstructTree(ctx, logClient, latestSTH)
|
||||
if isFatalLogError(err) {
|
||||
return err
|
||||
} else if err != nil {
|
||||
recordError(fmt.Errorf("error reconstructing tree of size %d for %s: %w", latestSTH.TreeSize, ctlog.URL, err))
|
||||
return nil
|
||||
}
|
||||
state = &stateFile{
|
||||
DownloadPosition: tree,
|
||||
VerifiedPosition: tree,
|
||||
VerifiedSTH: latestSTH,
|
||||
LastSuccess: startTime.UTC(),
|
||||
}
|
||||
} else {
|
||||
state = &stateFile{
|
||||
DownloadPosition: merkletree.EmptyCollapsedTree(),
|
||||
VerifiedPosition: merkletree.EmptyCollapsedTree(),
|
||||
VerifiedSTH: nil,
|
||||
LastSuccess: startTime.UTC(),
|
||||
}
|
||||
}
|
||||
if config.Verbose {
|
||||
log.Printf("brand new log %s (starting from %d)", ctlog.URL, state.DownloadPosition.Size())
|
||||
}
|
||||
if err := state.store(stateFilePath); err != nil {
|
||||
return fmt.Errorf("error storing state file: %w", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("error loading state file: %w", err)
|
||||
}
|
||||
|
||||
sths, err := loadSTHsFromDir(sthsDirPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error loading STHs directory: %w", err)
|
||||
}
|
||||
|
||||
for len(sths) > 0 && sths[0].TreeSize <= state.DownloadPosition.Size() {
|
||||
// TODO-4: audit sths[0] against state.VerifiedSTH
|
||||
if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
|
||||
return fmt.Errorf("error removing STH: %w", err)
|
||||
}
|
||||
sths = sths[1:]
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if config.Verbose {
|
||||
log.Printf("saving state in defer for %s", ctlog.URL)
|
||||
}
|
||||
if err := state.store(stateFilePath); err != nil && returnedErr == nil {
|
||||
returnedErr = fmt.Errorf("error storing state file: %w", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if len(sths) == 0 {
|
||||
state.LastSuccess = startTime.UTC()
|
||||
return nil
|
||||
}
|
||||
|
||||
var (
|
||||
downloadBegin = state.DownloadPosition.Size()
|
||||
downloadEnd = sths[len(sths)-1].TreeSize
|
||||
entries = make(chan client.GetEntriesItem, maxGetEntriesSize)
|
||||
downloadErr error
|
||||
)
|
||||
if config.Verbose {
|
||||
log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd)
|
||||
}
|
||||
go func() {
|
||||
defer close(entries)
|
||||
downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd)
|
||||
}()
|
||||
for rawEntry := range entries {
|
||||
entry := &logEntry{
|
||||
Log: ctlog,
|
||||
Index: state.DownloadPosition.Size(),
|
||||
LeafInput: rawEntry.LeafInput,
|
||||
ExtraData: rawEntry.ExtraData,
|
||||
LeafHash: merkletree.HashLeaf(rawEntry.LeafInput),
|
||||
}
|
||||
if err := processLogEntry(ctx, config, entry); err != nil {
|
||||
return fmt.Errorf("error processing entry %d: %w", entry.Index, err)
|
||||
}
|
||||
|
||||
state.DownloadPosition.Add(entry.LeafHash)
|
||||
rootHash := state.DownloadPosition.CalculateRoot()
|
||||
shouldSaveState := state.DownloadPosition.Size()%10000 == 0
|
||||
|
||||
for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize {
|
||||
if merkletree.Hash(sths[0].SHA256RootHash) != rootHash {
|
||||
recordError(fmt.Errorf("error verifying %s at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", ctlog.URL, sths[0].TreeSize, sths[0].SHA256RootHash, rootHash))
|
||||
|
||||
state.DownloadPosition = state.VerifiedPosition
|
||||
if err := state.store(stateFilePath); err != nil {
|
||||
return fmt.Errorf("error storing state file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
state.VerifiedPosition = state.DownloadPosition
|
||||
state.VerifiedSTH = sths[0]
|
||||
shouldSaveState = true
|
||||
if err := removeSTHFromDir(sthsDirPath, sths[0]); err != nil {
|
||||
return fmt.Errorf("error removing verified STH: %w", err)
|
||||
}
|
||||
|
||||
sths = sths[1:]
|
||||
}
|
||||
|
||||
if shouldSaveState {
|
||||
if err := state.store(stateFilePath); err != nil {
|
||||
return fmt.Errorf("error storing state file: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if isFatalLogError(downloadErr) {
|
||||
return downloadErr
|
||||
} else if downloadErr != nil {
|
||||
recordError(fmt.Errorf("error downloading entries from %s: %w", ctlog.URL, downloadErr))
|
||||
return nil
|
||||
}
|
||||
|
||||
if config.Verbose {
|
||||
log.Printf("finished downloading entries from %s", ctlog.URL)
|
||||
}
|
||||
|
||||
state.LastSuccess = startTime.UTC()
|
||||
return nil
|
||||
}
|
||||
|
||||
func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error {
|
||||
for begin < end && ctx.Err() == nil {
|
||||
size := begin - end
|
||||
if size > maxGetEntriesSize {
|
||||
size = maxGetEntriesSize
|
||||
}
|
||||
entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, entry := range entries {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case entriesChan <- entry:
|
||||
}
|
||||
}
|
||||
begin += uint64(len(entries))
|
||||
}
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) {
|
||||
if sth.TreeSize == 0 {
|
||||
return merkletree.EmptyCollapsedTree(), nil
|
||||
}
|
||||
entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
leafHash := merkletree.HashLeaf(entries[0].LeafInput)
|
||||
|
||||
var tree *merkletree.CollapsedTree
|
||||
if sth.TreeSize > 1 {
|
||||
auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hashes := make([]merkletree.Hash, len(auditPath))
|
||||
for i := range hashes {
|
||||
copy(hashes[i][:], auditPath[len(auditPath)-i-1])
|
||||
}
|
||||
tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err)
|
||||
}
|
||||
} else {
|
||||
tree = merkletree.EmptyCollapsedTree()
|
||||
}
|
||||
|
||||
tree.Add(leafHash)
|
||||
rootHash := tree.CalculateRoot()
|
||||
if rootHash != merkletree.Hash(sth.SHA256RootHash) {
|
||||
return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize)
|
||||
}
|
||||
|
||||
return tree, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var stdoutMu sync.Mutex
|
||||
|
||||
type notification interface {
|
||||
Environ() []string
|
||||
EmailSubject() string
|
||||
Text() string
|
||||
}
|
||||
|
||||
func notify(ctx context.Context, config *Config, notif notification) error {
|
||||
if config.Stdout {
|
||||
writeToStdout(notif)
|
||||
}
|
||||
|
||||
if len(config.Email) > 0 {
|
||||
if err := sendEmail(ctx, config.Email, notif); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if config.Script != "" {
|
||||
if err := execScript(ctx, config.Script, notif); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeToStdout(notif notification) {
|
||||
stdoutMu.Lock()
|
||||
defer stdoutMu.Unlock()
|
||||
os.Stdout.WriteString(notif.Text() + "\n")
|
||||
}
|
||||
|
||||
func sendEmail(ctx context.Context, to []string, notif notification) error {
|
||||
stdin := new(bytes.Buffer)
|
||||
stderr := new(bytes.Buffer)
|
||||
|
||||
fmt.Fprintf(stdin, "To: %s\n", strings.Join(to, ", "))
|
||||
fmt.Fprintf(stdin, "Subject: %s\n", notif.EmailSubject())
|
||||
fmt.Fprintf(stdin, "Mime-Version: 1.0\n")
|
||||
fmt.Fprintf(stdin, "Content-Type: text/plain; charset=US-ASCII\n")
|
||||
fmt.Fprintf(stdin, "X-Mailer: certspotter\n")
|
||||
fmt.Fprintf(stdin, "\n")
|
||||
fmt.Fprint(stdin, notif.Text())
|
||||
|
||||
args := []string{"-i", "--"}
|
||||
args = append(args, to...)
|
||||
|
||||
sendmail := exec.CommandContext(ctx, "/usr/sbin/sendmail", args...)
|
||||
sendmail.Stdin = stdin
|
||||
sendmail.Stderr = stderr
|
||||
|
||||
if err := sendmail.Run(); err == nil {
|
||||
return nil
|
||||
} else if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
|
||||
return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
|
||||
} else {
|
||||
return fmt.Errorf("error sending email to %v: %w", to, err)
|
||||
}
|
||||
}
|
||||
|
||||
func execScript(ctx context.Context, scriptPath string, notif notification) error {
|
||||
// TODO-3: consider removing directory support (for now), and supporting $PATH lookups
|
||||
info, err := os.Stat(scriptPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("script %q does not exist", scriptPath)
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("error executing script %q: %w", scriptPath, err)
|
||||
} else if info.IsDir() {
|
||||
return execScriptDir(ctx, scriptPath, notif)
|
||||
} else {
|
||||
return execScriptFile(ctx, scriptPath, notif)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func execScriptDir(ctx context.Context, dirPath string, notif notification) error {
|
||||
dirents, err := os.ReadDir(dirPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error executing scripts in directory %q: %w", dirPath, err)
|
||||
}
|
||||
for _, dirent := range dirents {
|
||||
if strings.HasPrefix(dirent.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
scriptPath := filepath.Join(dirPath, dirent.Name())
|
||||
info, err := os.Stat(scriptPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
continue
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("error executing %q in directory %q: %w", dirent.Name(), dirPath, err)
|
||||
} else if info.Mode().IsRegular() && isExecutable(info.Mode()) {
|
||||
if err := execScriptFile(ctx, scriptPath, notif); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func execScriptFile(ctx context.Context, scriptPath string, notif notification) error {
|
||||
stderr := new(bytes.Buffer)
|
||||
|
||||
cmd := exec.CommandContext(ctx, scriptPath)
|
||||
cmd.Env = os.Environ()
|
||||
cmd.Env = append(cmd.Env, notif.Environ()...)
|
||||
cmd.Stderr = stderr
|
||||
|
||||
if err := cmd.Run(); err == nil {
|
||||
return nil
|
||||
} else if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
|
||||
return fmt.Errorf("script %q exited with code %d and error %q", scriptPath, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
|
||||
} else if isExitError {
|
||||
return fmt.Errorf("script %q terminated by signal with error %q", scriptPath, strings.TrimSpace(stderr.String()))
|
||||
} else {
|
||||
return fmt.Errorf("error executing script %q: %w", scriptPath, err)
|
||||
}
|
||||
}
|
||||
|
||||
func isExecutable(mode os.FileMode) bool {
|
||||
return mode&0111 != 0
|
||||
}
|
||||
|
|
@ -0,0 +1,150 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"software.sslmate.com/src/certspotter"
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"software.sslmate.com/src/certspotter/loglist"
|
||||
"software.sslmate.com/src/certspotter/merkletree"
|
||||
)
|
||||
|
||||
type logEntry struct {
|
||||
Log *loglist.Log
|
||||
Index uint64
|
||||
LeafInput []byte
|
||||
ExtraData []byte
|
||||
LeafHash merkletree.Hash
|
||||
}
|
||||
|
||||
func processLogEntry(ctx context.Context, config *Config, entry *logEntry) error {
|
||||
leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput))
|
||||
if err != nil {
|
||||
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err))
|
||||
}
|
||||
switch leaf.TimestampedEntry.EntryType {
|
||||
case ct.X509LogEntryType:
|
||||
return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry)
|
||||
case ct.PrecertLogEntryType:
|
||||
return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry)
|
||||
default:
|
||||
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType))
|
||||
}
|
||||
}
|
||||
|
||||
func processX509LogEntry(ctx context.Context, config *Config, entry *logEntry, cert ct.ASN1Cert) error {
|
||||
certInfo, err := certspotter.MakeCertInfoFromRawCert(cert)
|
||||
if err != nil {
|
||||
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err))
|
||||
}
|
||||
|
||||
chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData)
|
||||
if err != nil {
|
||||
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err))
|
||||
}
|
||||
chain = append([]ct.ASN1Cert{cert}, chain...)
|
||||
|
||||
return processCertificate(ctx, config, entry, certInfo, chain)
|
||||
}
|
||||
|
||||
func processPrecertLogEntry(ctx context.Context, config *Config, entry *logEntry, precert ct.PreCert) error {
|
||||
certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate)
|
||||
if err != nil {
|
||||
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err))
|
||||
}
|
||||
|
||||
chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData)
|
||||
if err != nil {
|
||||
return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err))
|
||||
}
|
||||
|
||||
return processCertificate(ctx, config, entry, certInfo, chain)
|
||||
}
|
||||
|
||||
func processCertificate(ctx context.Context, config *Config, entry *logEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error {
|
||||
identifiers, err := certInfo.ParseIdentifiers()
|
||||
if err != nil {
|
||||
return processMalformedLogEntry(ctx, config, entry, err)
|
||||
}
|
||||
matched, watchItem := config.WatchList.Matches(identifiers)
|
||||
if !matched {
|
||||
return nil
|
||||
}
|
||||
|
||||
cert := &discoveredCert{
|
||||
WatchItem: watchItem,
|
||||
LogEntry: entry,
|
||||
Info: certInfo,
|
||||
Chain: chain,
|
||||
LeafSHA256: sha256.Sum256(chain[0]),
|
||||
Identifiers: identifiers,
|
||||
}
|
||||
|
||||
var notifiedPath string
|
||||
if config.SaveCerts {
|
||||
hexFingerprint := hex.EncodeToString(cert.LeafSHA256[:])
|
||||
prefixPath := filepath.Join(config.StateDir, "certs", hexFingerprint[0:2])
|
||||
|
||||
for _, suffix := range []string{".notified", ".cert.pem", ".precert.pem"} {
|
||||
if fileExists(filepath.Join(prefixPath, hexFingerprint+suffix)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.Mkdir(prefixPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return fmt.Errorf("error creating directory in which to save certificate %x: %w", cert.LeafSHA256, err)
|
||||
}
|
||||
|
||||
notifiedPath = filepath.Join(prefixPath, hexFingerprint+".notified") // TODO-3: maybe this should be a hidden file?
|
||||
cert.CertPath = filepath.Join(prefixPath, hexFingerprint+".pem")
|
||||
cert.JSONPath = filepath.Join(prefixPath, hexFingerprint+".json") // TODO-3: consider using .v1.json extension in case I change the format later?
|
||||
cert.TextPath = filepath.Join(prefixPath, hexFingerprint+".txt")
|
||||
|
||||
if err := cert.save(); err != nil {
|
||||
return fmt.Errorf("error saving certificate %x: %w", cert.LeafSHA256, err)
|
||||
}
|
||||
} else {
|
||||
// TODO-4: save cert to temporary files, and defer their unlinking
|
||||
}
|
||||
|
||||
if err := notify(ctx, config, cert); err != nil {
|
||||
return fmt.Errorf("error notifying about discovered certificate for %s (%x): %w", cert.WatchItem, cert.LeafSHA256, err)
|
||||
}
|
||||
|
||||
if notifiedPath != "" {
|
||||
if err := os.WriteFile(notifiedPath, nil, 0666); err != nil {
|
||||
return fmt.Errorf("error saving certificate %x: %w", cert.LeafSHA256, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processMalformedLogEntry(ctx context.Context, config *Config, entry *logEntry, parseError error) error {
|
||||
// TODO-4: save the malformed entry (in get-entries format) in the state directory so user can inspect it
|
||||
|
||||
malformed := &malformedLogEntry{
|
||||
Entry: entry,
|
||||
Error: parseError.Error(),
|
||||
}
|
||||
if err := notify(ctx, config, malformed); err != nil {
|
||||
return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,155 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"software.sslmate.com/src/certspotter/merkletree"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
func readVersion(stateDir string) (int, error) {
|
||||
path := filepath.Join(stateDir, "version")
|
||||
|
||||
fileBytes, err := os.ReadFile(path)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
if fileExists(filepath.Join(stateDir, "evidence")) {
|
||||
return 0, nil
|
||||
} else {
|
||||
return -1, nil
|
||||
}
|
||||
} else if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
version, err := strconv.Atoi(strings.TrimSpace(string(fileBytes)))
|
||||
if err != nil {
|
||||
return -1, fmt.Errorf("version file %q is malformed: %w", path, err)
|
||||
}
|
||||
|
||||
return version, nil
|
||||
}
|
||||
|
||||
func writeVersion(stateDir string) error {
|
||||
return writeFile(filepath.Join(stateDir, "version"), []byte{'2', '\n'}, 0666)
|
||||
}
|
||||
|
||||
func migrateLogStateDirV1(dir string) error {
|
||||
var sth ct.SignedTreeHead
|
||||
var tree merkletree.CollapsedTree
|
||||
|
||||
sthPath := filepath.Join(dir, "sth.json")
|
||||
sthData, err := os.ReadFile(sthPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
treePath := filepath.Join(dir, "tree.json")
|
||||
treeData, err := os.ReadFile(treePath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := json.Unmarshal(sthData, &sth); err != nil {
|
||||
return fmt.Errorf("error unmarshaling %s: %w", sthPath, err)
|
||||
}
|
||||
if err := json.Unmarshal(treeData, &tree); err != nil {
|
||||
return fmt.Errorf("error unmarshaling %s: %w", treePath, err)
|
||||
}
|
||||
|
||||
stateFile := stateFile{
|
||||
DownloadPosition: &tree,
|
||||
VerifiedPosition: &tree,
|
||||
VerifiedSTH: &sth,
|
||||
LastSuccess: time.Now().UTC(),
|
||||
}
|
||||
if stateFile.store(filepath.Join(dir, "state.json")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Remove(sthPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Remove(treePath); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateStateDirV1(stateDir string) error {
|
||||
if lockfile := filepath.Join(stateDir, "lock"); fileExists(lockfile) {
|
||||
return fmt.Errorf("directory is locked by another instance of certspotter; remove %s if this is not the case", lockfile)
|
||||
}
|
||||
|
||||
if logDirs, err := os.ReadDir(filepath.Join(stateDir, "logs")); err == nil {
|
||||
for _, logDir := range logDirs {
|
||||
if strings.HasPrefix(logDir.Name(), ".") || !logDir.IsDir() {
|
||||
continue
|
||||
}
|
||||
if err := migrateLogStateDirV1(filepath.Join(stateDir, "logs", logDir.Name())); err != nil {
|
||||
return fmt.Errorf("error migrating log state: %w", err)
|
||||
}
|
||||
}
|
||||
} else if !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := writeVersion(stateDir); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := os.Remove(filepath.Join(stateDir, "once")); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func prepareStateDir(stateDir string) error {
|
||||
if err := os.Mkdir(stateDir, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return err
|
||||
}
|
||||
|
||||
if version, err := readVersion(stateDir); err != nil {
|
||||
return err
|
||||
} else if version == -1 {
|
||||
if err := writeVersion(stateDir); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if version == 0 {
|
||||
return fmt.Errorf("%s was created by a very old version of certspotter; run any version of certspotter after 0.2 and before 0.15.0 to upgrade this directory, or remove it to start from scratch", stateDir)
|
||||
} else if version == 1 {
|
||||
if err := migrateStateDirV1(stateDir); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if version > 2 {
|
||||
return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
|
||||
}
|
||||
|
||||
for _, subdir := range []string{"certs", "logs"} {
|
||||
if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
// Copyright (C) 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"software.sslmate.com/src/certspotter/merkletree"
|
||||
"time"
|
||||
)
|
||||
|
||||
type stateFile struct {
|
||||
DownloadPosition *merkletree.CollapsedTree `json:"download_position"`
|
||||
VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"`
|
||||
VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"`
|
||||
LastSuccess time.Time `json:"last_success"`
|
||||
}
|
||||
|
||||
func loadStateFile(filePath string) (*stateFile, error) {
|
||||
fileBytes, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
file := new(stateFile)
|
||||
if err := json.Unmarshal(fileBytes, file); err != nil {
|
||||
return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
|
||||
}
|
||||
return file, nil
|
||||
}
|
||||
|
||||
func (file *stateFile) store(filePath string) error {
|
||||
fileBytes, err := json.Marshal(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fileBytes = append(fileBytes, '\n')
|
||||
return writeFile(filePath, fileBytes, 0666)
|
||||
}
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
// Copyright (C) 2017, 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"golang.org/x/exp/slices"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"software.sslmate.com/src/certspotter/ct"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) {
|
||||
entries, err := os.ReadDir(dirPath)
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return []*ct.SignedTreeHead{}, nil
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sths := make([]*ct.SignedTreeHead, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
filename := entry.Name()
|
||||
if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") {
|
||||
continue
|
||||
}
|
||||
sth, err := readSTHFile(filepath.Join(dirPath, filename))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sths = append(sths, sth)
|
||||
}
|
||||
slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) bool { return a.TreeSize < b.TreeSize })
|
||||
return sths, nil
|
||||
}
|
||||
|
||||
func readSTHFile(filePath string) (*ct.SignedTreeHead, error) {
|
||||
fileBytes, err := os.ReadFile(filePath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sth := new(ct.SignedTreeHead)
|
||||
if err := json.Unmarshal(fileBytes, sth); err != nil {
|
||||
return nil, fmt.Errorf("error parsing %s: %w", filePath, err)
|
||||
}
|
||||
return sth, nil
|
||||
}
|
||||
|
||||
func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error {
|
||||
filePath := filepath.Join(dirPath, sthFilename(sth))
|
||||
if fileExists(filePath) {
|
||||
return nil
|
||||
}
|
||||
fileBytes, err := json.Marshal(sth)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return writeFile(filePath, fileBytes, 0666)
|
||||
}
|
||||
|
||||
func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error {
|
||||
filePath := filepath.Join(dirPath, sthFilename(sth))
|
||||
err := os.Remove(filePath)
|
||||
if err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// generate a filename that uniquely identifies the STH (within the context of a particular log)
|
||||
func sthFilename(sth *ct.SignedTreeHead) string {
|
||||
hasher := sha256.New()
|
||||
switch sth.Version {
|
||||
case ct.V1:
|
||||
binary.Write(hasher, binary.LittleEndian, sth.Timestamp)
|
||||
binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash)
|
||||
default:
|
||||
panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version))
|
||||
}
|
||||
// For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic)
|
||||
return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json"
|
||||
}
|
||||
|
|
@ -0,0 +1,135 @@
|
|||
// Copyright (C) 2016, 2023 Opsmate, Inc.
|
||||
//
|
||||
// This Source Code Form is subject to the terms of the Mozilla
|
||||
// Public License, v. 2.0. If a copy of the MPL was not distributed
|
||||
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
//
|
||||
// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
|
||||
// See the Mozilla Public License for details.
|
||||
|
||||
package monitor
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"golang.org/x/net/idna"
|
||||
"io"
|
||||
"software.sslmate.com/src/certspotter"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type WatchItem struct {
|
||||
domain []string
|
||||
acceptSuffix bool
|
||||
}
|
||||
|
||||
type WatchList []WatchItem
|
||||
|
||||
func ParseWatchItem(str string) (WatchItem, error) {
|
||||
fields := strings.Fields(str)
|
||||
if len(fields) == 0 {
|
||||
return WatchItem{}, fmt.Errorf("empty domain")
|
||||
}
|
||||
domain := fields[0]
|
||||
|
||||
for _, field := range fields[1:] {
|
||||
switch {
|
||||
case strings.HasPrefix(field, "valid_at:"):
|
||||
// Ignore for backwards compatibility
|
||||
default:
|
||||
return WatchItem{}, fmt.Errorf("unknown parameter %q", field)
|
||||
}
|
||||
}
|
||||
|
||||
if domain == "." {
|
||||
// "." as in root zone -> matches everything
|
||||
return WatchItem{
|
||||
domain: []string{},
|
||||
acceptSuffix: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
acceptSuffix := false
|
||||
if strings.HasPrefix(domain, ".") {
|
||||
acceptSuffix = true
|
||||
domain = domain[1:]
|
||||
}
|
||||
|
||||
asciiDomain, err := idna.ToASCII(strings.ToLower(strings.TrimRight(domain, ".")))
|
||||
if err != nil {
|
||||
return WatchItem{}, fmt.Errorf("invalid domain %q (%w)", domain, err)
|
||||
}
|
||||
return WatchItem{
|
||||
domain: strings.Split(asciiDomain, "."),
|
||||
acceptSuffix: acceptSuffix,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func ReadWatchList(reader io.Reader) (WatchList, error) {
|
||||
items := make(WatchList, 0, 50)
|
||||
scanner := bufio.NewScanner(reader)
|
||||
lineNo := 0
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
lineNo++
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
item, err := ParseWatchItem(line)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w on line %d", err, lineNo)
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
return items, scanner.Err()
|
||||
}
|
||||
|
||||
func (item WatchItem) String() string {
|
||||
if item.acceptSuffix {
|
||||
return "." + strings.Join(item.domain, ".")
|
||||
} else {
|
||||
return strings.Join(item.domain, ".")
|
||||
}
|
||||
}
|
||||
|
||||
func (item WatchItem) matchesDNSName(dnsName []string) bool {
|
||||
watchDomain := item.domain
|
||||
for len(dnsName) > 0 && len(watchDomain) > 0 {
|
||||
certLabel := dnsName[len(dnsName)-1]
|
||||
watchLabel := watchDomain[len(watchDomain)-1]
|
||||
|
||||
if !dnsLabelMatches(certLabel, watchLabel) {
|
||||
return false
|
||||
}
|
||||
|
||||
dnsName = dnsName[:len(dnsName)-1]
|
||||
watchDomain = watchDomain[:len(watchDomain)-1]
|
||||
}
|
||||
return len(watchDomain) == 0 && (item.acceptSuffix || len(dnsName) == 0)
|
||||
}
|
||||
|
||||
func dnsLabelMatches(certLabel string, watchLabel string) bool {
|
||||
// For fail-safe behavior, if a label was unparsable, it matches everything.
|
||||
// Similarly, redacted labels match everything, since the label _might_ be
|
||||
// for a name we're interested in.
|
||||
|
||||
return certLabel == "*" ||
|
||||
certLabel == "?" ||
|
||||
certLabel == certspotter.UnparsableDNSLabelPlaceholder ||
|
||||
certspotter.MatchesWildcard(watchLabel, certLabel)
|
||||
}
|
||||
|
||||
func (list WatchList) Matches(identifiers *certspotter.Identifiers) (bool, WatchItem) {
|
||||
dnsNames := make([][]string, len(identifiers.DNSNames))
|
||||
for i, dnsName := range identifiers.DNSNames {
|
||||
dnsNames[i] = strings.Split(dnsName, ".")
|
||||
}
|
||||
for _, item := range list {
|
||||
for _, dnsName := range dnsNames {
|
||||
if item.matchesDNSName(dnsName) {
|
||||
return true, item
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, WatchItem{}
|
||||
}
|
||||
Loading…
Reference in New Issue