Release v0.21.0

These are important and should not happen very often.

.github/workflows/test.yml

@@ -0,0 +1,35 @@
+name: Test and lint Go Code
+
+on:
+  push:
+  schedule:
+    - cron: '42 9 * * *' # Runs daily at 09:42 UTC
+  workflow_dispatch:     # Allows manual triggering
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Run tests
+        run: CGO_ENABLED=1 go test -race ./...
+
+      - name: Install staticcheck
+        run: go install honnef.co/go/tools/cmd/staticcheck@latest
+
+      - name: Run staticcheck
+        run: staticcheck ./...

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Change Log
 
+## v0.21.0 (2025-07-02)
+- Instead of writing log errors to stderr, save the last 7 days worth in the
+  state directory, and include recent errors in failed health check notifications.
+- Send a meaningful User-Agent string with HTTP requests.
+
+## v0.20.1 (2025-06-19)
+- Add resilience against sendmail hanging indefinitely.
+- Add resilience against hooks which fork and keep stderr open.
+- Upgrade dependencies to latest versions.
+- Minor improvements to error handling, code quality, and efficiency.
+
 ## v0.20.0 (2025-06-13)
 - Remove -batch_size option, which is obsolete due to new parallel download system.
 - Only print log errors to stderr if -verbose is specified.

asn1.go

@@ -46,7 +46,7 @@ func decodeASN1String(value *asn1.RawValue) (string, error) {
 		if value.Tag == 12 {
 			// UTF8String
 			if !utf8.Valid(value.Bytes) {
-				return "", errors.New("Malformed UTF8String")
+				return "", errors.New("malformed UTF8String")
 			}
 			return string(value.Bytes), nil
 		} else if value.Tag == 19 || value.Tag == 22 || value.Tag == 20 || value.Tag == 26 {
@@ -74,5 +74,5 @@ func decodeASN1String(value *asn1.RawValue) (string, error) {
 			return stringFromUint32Slice(runes), nil
 		}
 	}
-	return "", errors.New("Not a string")
+	return "", errors.New("not a string")
 }

asn1time.go

@@ -253,5 +253,5 @@ func decodeASN1Time(value *asn1.RawValue) (time.Time, error) {
 			return parseGeneralizedTime(value.Bytes)
 		}
 	}
-	return time.Time{}, errors.New("Not a time value")
+	return time.Time{}, errors.New("not a time value")
 }

cmd/certspotter/main.go

@@ -25,43 +25,23 @@ import (
 	"syscall"
 	"time"
 
+	"software.sslmate.com/src/certspotter/ctclient"
 	"software.sslmate.com/src/certspotter/loglist"
 	"software.sslmate.com/src/certspotter/monitor"
 )
 
 var programName = os.Args[0]
-var Version = ""
+var Version = "unknown"
+var Source = "unknown"
 
 const defaultLogList = "https://loglist.certspotter.org/monitor.json"
 
-func certspotterVersion() string {
-	if Version != "" {
-		return Version + "?"
+func certspotterVersion() (string, string) {
+	if buildinfo, ok := debug.ReadBuildInfo(); ok && strings.HasPrefix(buildinfo.Main.Version, "v") {
+		return strings.TrimPrefix(buildinfo.Main.Version, "v"), buildinfo.Main.Path
+	} else {
+		return Version, Source
 	}
-	info, ok := debug.ReadBuildInfo()
-	if !ok {
-		return "unknown"
-	}
-	if strings.HasPrefix(info.Main.Version, "v") {
-		return info.Main.Version
-	}
-	var vcs, vcsRevision, vcsModified string
-	for _, s := range info.Settings {
-		switch s.Key {
-		case "vcs":
-			vcs = s.Value
-		case "vcs.revision":
-			vcsRevision = s.Value
-		case "vcs.modified":
-			vcsModified = s.Value
-		}
-	}
-	if vcs == "git" && vcsRevision != "" && vcsModified == "true" {
-		return vcsRevision + "+"
-	} else if vcs == "git" && vcsRevision != "" {
-		return vcsRevision
-	}
-	return "unknown"
 }
 
 func fileExists(filename string) bool {
@@ -158,7 +138,10 @@ func appendFunc(slice *[]string) func(string) error {
 }
 
 func main() {
-	loglist.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s)", certspotterVersion(), runtime.Version(), runtime.GOOS, runtime.GOARCH)
+	version, source := certspotterVersion()
+
+	ctclient.UserAgent = fmt.Sprintf("certspotter/%s (%s; %s; %s; %s; +https://github.com/SSLMate/certspotter)", version, source, runtime.Version(), runtime.GOOS, runtime.GOARCH)
+	loglist.UserAgent = ctclient.UserAgent
 
 	var flags struct {
 		batchSize   bool
@@ -193,7 +176,7 @@ func main() {
 		os.Exit(2)
 	}
 	if flags.version {
-		fmt.Fprintf(os.Stdout, "certspotter version %s\n", certspotterVersion())
+		fmt.Fprintf(os.Stdout, "certspotter version %s (%s)\n", version, source)
 		os.Exit(0)
 	}
 	if flags.watchlist == "" {
@@ -209,7 +192,6 @@ func main() {
 		ScriptDir: defaultScriptDir(),
 		Email:     flags.email,
 		Stdout:    flags.stdout,
-		Quiet:     !flags.verbose,
 	}
 	config := &monitor.Config{
 		LogListSource:       flags.logs,
@@ -258,6 +240,19 @@ func main() {
 	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
 	defer stop()
 
+	go func() {
+		ticker := time.NewTicker(24*time.Hour)
+		defer ticker.Stop()
+		for {
+			fsstate.PruneOldErrors()
+			select {
+			case <-ctx.Done():
+				return
+			case <-ticker.C:
+			}
+		}
+	}()
+
 	if err := monitor.Run(ctx, config); ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) {
 		if flags.verbose {
 			fmt.Fprintf(os.Stderr, "%s: exiting due to SIGINT or SIGTERM\n", programName)

ctclient/client.go

@@ -24,6 +24,8 @@ import (
 	"time"
 )
 
+var UserAgent = ""
+
 // Create an HTTP client suitable for communicating with CT logs.  dialContext, if non-nil, is used for dialing.
 func NewHTTPClient(dialContext func(context.Context, string, string) (net.Conn, error)) *http.Client {
 	return &http.Client{
@@ -61,7 +63,7 @@ func get(ctx context.Context, httpClient *http.Client, fullURL string) ([]byte,
 	if err != nil {
 		return nil, err
 	}
-	request.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs
+	request.Header.Set("User-Agent", UserAgent)
 
 	if httpClient == nil {
 		httpClient = defaultHTTPClient

cttypes/logid.go

@@ -10,6 +10,7 @@
 package cttypes
 
 import (
+	"bytes"
 	"encoding/base64"
 	"fmt"
 	"golang.org/x/crypto/cryptobyte"
@@ -17,6 +18,10 @@ import (
 
 type LogID [32]byte
 
+func (id LogID) Compare(other LogID) int {
+	return bytes.Compare(id[:], other[:])
+}
+
 func (v *LogID) Unmarshal(s *cryptobyte.String) bool {
 	return s.CopyBytes((*v)[:])
 }

go.mod

@@ -1,13 +1,13 @@
 module software.sslmate.com/src/certspotter
 
-go 1.24
+go 1.24.4
 
 require (
-	golang.org/x/crypto v0.37.0
-	golang.org/x/net v0.39.0
-	golang.org/x/sync v0.13.0
+	golang.org/x/crypto v0.39.0
+	golang.org/x/net v0.41.0
+	golang.org/x/sync v0.15.0
 )
 
-require golang.org/x/text v0.24.0 // indirect
+require golang.org/x/text v0.26.0 // indirect
 
 retract v0.19.0 // Contains serious bugs.

go.sum

@@ -1,8 +1,8 @@
-golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
-golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
-golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY=
-golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E=
-golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
-golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
-golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
-golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
+golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
+golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
+golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw=
+golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA=
+golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8=
+golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M=
+golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA=

loglist/load.go

@@ -21,7 +21,7 @@ import (
 	"time"
 )
 
-var UserAgent = "certspotter"
+var UserAgent = ""
 
 type ModificationToken struct {
 	etag     string
@@ -112,7 +112,7 @@ func Unmarshal(jsonBytes []byte) (*List, error) {
 		return nil, err
 	}
 	if err := list.Validate(); err != nil {
-		return nil, fmt.Errorf("Invalid log list: %s", err)
+		return nil, fmt.Errorf("invalid log list: %s", err)
 	}
 	return list, nil
 }

man/certspotter.md

@@ -53,8 +53,8 @@ You can use Cert Spotter to detect:
 :   Filename or HTTPS URL of a v2 or v3 JSON log list containing logs to monitor.
     The schema for this file can be found at <https://www.gstatic.com/ct/log_list/v3/log_list_schema.json>.
     Defaults to <https://loglist.certspotter.org/monitor.json>, which includes
-    the union of active logs recognized by Chrome and Apple.  certspotter periodically
-    reloads the log list in case it has changed.
+    the union of active logs recognized by Chrome and Apple.  certspotter loads the
+    log list when starting up, and periodically reloads it in case it has changed.
 
 -no\_save
 
@@ -90,7 +90,7 @@ You can use Cert Spotter to detect:
 
 -verbose
 
-:   Print detailed information about certspotter's operation (such as errors contacting logs) to stderr.
+:   Print detailed information about certspotter's operation to stderr.
 
 -version
 
@@ -136,7 +136,7 @@ the script interface, see certspotter-script(8).
 # OPERATION
 
 certspotter continuously monitors all browser-recognized Certificate
-Transparency logs looking for certificates (including precertificates)
+Transparency logs (both RFC6962 and static-ct-api) looking for certificates (including precertificates)
 which are valid for any domain on your watch list. When certspotter
 detects a matching certificate, it emails you, executes a script, and/or
 writes a report to standard out, as described above.
@@ -169,12 +169,17 @@ API <https://sslmate.com/ct_search_api>, or a CT search engine such as
 # ERROR HANDLING
 
 When certspotter encounters a problem with the local system (e.g. failure
-to write a file or execute a script), it prints a message to stderr and
+to write a file, send an email, or execute a script), it prints a message to stderr and
 exits with a non-zero status.
 
-When certspotter encounters a problem monitoring a log, it prints a message
-to stderr if `-verbose` is specified and continues running.  It will try monitoring the log again later;
-most log errors are transient.
+When certspotter encounters a problem loading the log list during startup, it
+prints a message to stderr and exits with a non-zero status.  When certspotter encounters a problem
+reloading the log list, it prints a message to stderr and continues running with the previously-loaded
+log list.  It will try reloading the log list again later.
+
+When certspotter encounters a problem contacting a log, it writes the error to a file in
+the state directory and continues running.  It will try contacting the log again later;
+most log errors are transient.  The last 7 days of errors are kept.
 
 Every 24 hours (unless overridden by `-healthcheck`), certspotter performs the
 following health checks:
@@ -186,11 +191,12 @@ following health checks:
  * Ensure that certspotter is not falling behind monitoring any logs.
 
 If any health check fails, certspotter notifies you by email, script, and/or
-standard out, as described above.
+standard out, as described above.  The notification includes the last several errors
+encountered when contacting the log.
 
 Health check failures should be rare, and you should take them seriously because it means
 certspotter might not detect all certificates.  It might also be an indication
-of CT log misbehavior.  Enable the `-verbose` flag and consult stderr for details, and if
+of CT log misbehavior.  Check the error files for details, and if
 you need help, file an issue at <https://github.com/SSLMate/certspotter>.
 
 # EXIT STATUS

merkletree/hash.go

@@ -10,6 +10,7 @@
 package merkletree
 
 import (
+	"bytes"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/json"
@@ -20,6 +21,10 @@ const HashLen = 32
 
 type Hash [HashLen]byte
 
+func (h Hash) Compare(other Hash) int {
+	return bytes.Compare(h[:], other[:])
+}
+
 func (h Hash) Base64String() string {
 	return base64.StdEncoding.EncodeToString(h[:])
 }

monitor/daemon.go

@@ -44,17 +44,23 @@ type daemon struct {
 	tasks          map[LogID]task
 	logsLoadedAt   time.Time
 	logListToken   *loglist.ModificationToken
-	logListError   string
-	logListErrorAt time.Time
 }
 
 func (daemon *daemon) healthCheck(ctx context.Context) error {
 	if time.Since(daemon.logsLoadedAt) >= daemon.config.HealthCheckInterval {
+		errors, err := daemon.config.State.GetErrors(ctx, nil, recentErrorCount)
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
+		var errorsDir string
+		if fsstate, ok := daemon.config.State.(*FilesystemState); ok {
+			errorsDir = fsstate.errorDir(nil)
+		}
 		info := &StaleLogListInfo{
 			Source:        daemon.config.LogListSource,
 			LastSuccess:   daemon.logsLoadedAt,
-			LastError:     daemon.logListError,
-			LastErrorTime: daemon.logListErrorAt,
+			RecentErrors:  errors,
+			ErrorsDir:     errorsDir,
 		}
 		if err := daemon.config.State.NotifyHealthCheckFailure(ctx, nil, info); err != nil {
 			return fmt.Errorf("error notifying about stale log list: %w", err)
@@ -143,8 +149,6 @@ func (daemon *daemon) run(ctx context.Context) error {
 			return ctx.Err()
 		case <-reloadLogListTicker.C:
 			if err := daemon.loadLogList(ctx); err != nil {
-				daemon.logListError = err.Error()
-				daemon.logListErrorAt = time.Now()
 				recordError(ctx, daemon.config, nil, fmt.Errorf("error reloading log list (will try again later): %w", err))
 			}
 			reloadLogListTicker.Reset(reloadLogListInterval())

monitor/fileutils.go

@@ -14,7 +14,9 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
+	"slices"
 )
 
 func randomFileSuffix() string {
@@ -69,3 +71,47 @@ func fileExists(filename string) bool {
 	_, err := os.Lstat(filename)
 	return err == nil
 }
+
+func tailFile(filename string, linesWanted int) ([]byte, int, error) {
+	file, err := os.Open(filename)
+	if err != nil {
+		return nil, 0, err
+	}
+	defer file.Close()
+	return tail(file, linesWanted, 4096)
+}
+
+func tail(r io.ReadSeeker, linesWanted int, chunkSize int) ([]byte, int, error) {
+	var buf []byte
+	linesGot := 0
+
+	offset, err := r.Seek(0, io.SeekEnd)
+	if err != nil {
+		return nil, 0, err
+	}
+	for offset > 0 {
+		readSize := chunkSize
+		if offset < int64(readSize) {
+			readSize = int(offset)
+		}
+		offset -= int64(readSize)
+		if _, err := r.Seek(offset, io.SeekStart); err != nil {
+			return nil, 0, err
+		}
+		buf = slices.Grow(buf, readSize)
+		copy(buf[readSize:len(buf)+readSize], buf)
+		buf = buf[:len(buf)+readSize]
+		if _, err := io.ReadFull(r, buf[:readSize]); err != nil {
+			return nil, 0, err
+		}
+		for i := readSize; i > 0; i-- {
+			if buf[i-1] == '\n' {
+				if linesGot == linesWanted {
+					return buf[i:], linesGot, nil
+				}
+				linesGot++
+			}
+		}
+	}
+	return buf, linesGot, nil
+}

monitor/fsstate.go

@@ -20,12 +20,17 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"time"
 
 	"software.sslmate.com/src/certspotter/cttypes"
 	"software.sslmate.com/src/certspotter/loglist"
 	"software.sslmate.com/src/certspotter/merkletree"
 )
 
+const keepErrorDays = 7
+const errorDateFormat = "2006-01-02"
+
 type FilesystemState struct {
 	StateDir  string
 	CacheDir  string
@@ -34,7 +39,7 @@ type FilesystemState struct {
 	ScriptDir string
 	Email     []string
 	Stdout    bool
-	Quiet     bool
+	errorMu   sync.Mutex
 }
 
 func (s *FilesystemState) logStateDir(logID LogID) string {
@@ -57,8 +62,9 @@ func (s *FilesystemState) PrepareLog(ctx context.Context, logID LogID) error {
 		sthsDirPath         = filepath.Join(stateDirPath, "unverified_sths")
 		malformedDirPath    = filepath.Join(stateDirPath, "malformed_entries")
 		healthchecksDirPath = filepath.Join(stateDirPath, "healthchecks")
+		errorsDirPath       = filepath.Join(stateDirPath, "errors")
 	)
-	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath} {
+	for _, dirPath := range []string{stateDirPath, sthsDirPath, malformedDirPath, healthchecksDirPath, errorsDirPath} {
 		if err := os.Mkdir(dirPath, 0777); err != nil && !errors.Is(err, fs.ErrExist) {
 			return err
 		}
@@ -227,6 +233,13 @@ func (s *FilesystemState) healthCheckDir(ctlog *loglist.Log) string {
 	}
 }
 
+func (s *FilesystemState) errorDir(ctlog *loglist.Log) string {
+	if ctlog == nil {
+		return filepath.Join(s.StateDir, "errors")
+	}
+	return filepath.Join(s.logStateDir(ctlog.LogID), "errors")
+}
+
 func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *loglist.Log, info HealthCheckFailure) error {
 	textPath := filepath.Join(s.healthCheckDir(ctlog), healthCheckFilename())
 	environ := []string{
@@ -248,13 +261,84 @@ func (s *FilesystemState) NotifyHealthCheckFailure(ctx context.Context, ctlog *l
 	return nil
 }
 
-func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, err error) error {
-	if !s.Quiet {
-		if ctlog == nil {
-			log.Print(err)
-		} else {
-			log.Print(ctlog.GetMonitoringURL(), ": ", err)
+func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, notifyErr error) error {
+	if ctlog == nil {
+		log.Print(notifyErr)
+	}
+
+	var (
+		now      = time.Now()
+		filePath = filepath.Join(s.errorDir(ctlog), now.Format(errorDateFormat))
+		line     = now.Format(time.RFC3339) + " " + notifyErr.Error() + "\n"
+	)
+
+	s.errorMu.Lock()
+	defer s.errorMu.Unlock()
+	file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+	if _, err := file.WriteString(line); err != nil {
+		return err
+	}
+	return file.Close()
+}
+
+func (s *FilesystemState) GetErrors(ctx context.Context, ctlog *loglist.Log, count int) (string, error) {
+	dir := s.errorDir(ctlog)
+	now := time.Now()
+	var buf []byte
+	for daysBack := 0; count > 0 && daysBack < keepErrorDays; daysBack++ {
+		datePath := filepath.Join(dir, now.AddDate(0, 0, -daysBack).Format(errorDateFormat))
+		dateBuf, dateLines, err := tailFile(datePath, count)
+		if errors.Is(err, fs.ErrNotExist) {
+			continue
+		} else if err != nil {
+			return "", err
+		}
+		buf = append(dateBuf, buf...)
+		count -= dateLines
+	}
+	return string(buf), nil
+}
+
+func (s *FilesystemState) PruneOldErrors() {
+	cutoff := time.Now().AddDate(0, 0, -keepErrorDays)
+	pruneDir := func(dir string) {
+		entries, err := os.ReadDir(dir)
+		if errors.Is(err, fs.ErrNotExist) {
+			return
+		} else if err != nil {
+			log.Printf("unable to read error directory: %s", err)
+			return
+		}
+		for _, entry := range entries {
+			if entry.IsDir() {
+				continue
+			}
+			date, err := time.Parse(errorDateFormat, entry.Name())
+			if err != nil {
+				continue
+			}
+			if date.Before(cutoff) {
+				if err := os.Remove(filepath.Join(dir, entry.Name())); err != nil && !errors.Is(err, fs.ErrNotExist) {
+					log.Printf("unable to remove old error file: %s", err)
+				}
+			}
 		}
 	}
-	return nil
+	pruneDir(filepath.Join(s.StateDir, "errors"))
+	logsDir := filepath.Join(s.StateDir, "logs")
+	logDirs, err := os.ReadDir(logsDir)
+	if err != nil && !errors.Is(err, fs.ErrNotExist) {
+		log.Printf("unable to read logs directory: %s", err)
+		return
+	}
+	for _, d := range logDirs {
+		if !d.IsDir() {
+			continue
+		}
+		pruneDir(filepath.Join(logsDir, d.Name(), "errors"))
+	}
 }

monitor/healthcheck.go

@@ -19,6 +19,8 @@ import (
 	"software.sslmate.com/src/certspotter/loglist"
 )
 
+const recentErrorCount = 10
+
 func healthCheckFilename() string {
 	return time.Now().UTC().Format(time.RFC3339) + ".txt"
 }
@@ -48,20 +50,37 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err
 		return fmt.Errorf("error loading STHs: %w", err)
 	}
 
+	var errorsDir string
+	if fsstate, ok := config.State.(*FilesystemState); ok {
+		errorsDir = fsstate.errorDir(ctlog)
+	}
+
 	if len(sths) == 0 {
+		errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount)
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
 		info := &StaleSTHInfo{
-			Log:         ctlog,
-			LastSuccess: lastSuccess,
-			LatestSTH:   verifiedSTH,
+			Log:          ctlog,
+			LastSuccess:  lastSuccess,
+			LatestSTH:    verifiedSTH,
+			RecentErrors: errors,
+			ErrorsDir:    errorsDir,
 		}
 		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
 			return fmt.Errorf("error notifying about stale STH: %w", err)
 		}
 	} else {
+		errors, err := config.State.GetErrors(ctx, ctlog, recentErrorCount)
+		if err != nil {
+			return fmt.Errorf("error getting recent errors: %w", err)
+		}
 		info := &BacklogInfo{
-			Log:       ctlog,
-			LatestSTH: sths[len(sths)-1],
-			Position:  position,
+			Log:          ctlog,
+			LatestSTH:    sths[len(sths)-1],
+			Position:     position,
+			RecentErrors: errors,
+			ErrorsDir:    errorsDir,
 		}
 		if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil {
 			return fmt.Errorf("error notifying about backlog: %w", err)
@@ -77,22 +96,26 @@ type HealthCheckFailure interface {
 }
 
 type StaleSTHInfo struct {
-	Log         *loglist.Log
-	LastSuccess time.Time               // may be zero
-	LatestSTH   *cttypes.SignedTreeHead // may be nil
+	Log          *loglist.Log
+	LastSuccess  time.Time               // may be zero
+	LatestSTH    *cttypes.SignedTreeHead // may be nil
+	RecentErrors string
+	ErrorsDir    string
 }
 
 type BacklogInfo struct {
-	Log       *loglist.Log
-	LatestSTH *StoredSTH
-	Position  uint64
+	Log          *loglist.Log
+	LatestSTH    *StoredSTH
+	Position     uint64
+	RecentErrors string
+	ErrorsDir    string
 }
 
 type StaleLogListInfo struct {
 	Source        string
 	LastSuccess   time.Time
-	LastError     string
-	LastErrorTime time.Time
+	RecentErrors  string
+	ErrorsDir     string
 }
 
 func (e *StaleSTHInfo) LastSuccessString() string {
@@ -120,33 +143,45 @@ func (e *StaleSTHInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL(), e.LastSuccessString())
 	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "For details, enable -verbose and see certspotter's stderr output.\n")
-	fmt.Fprintf(text, "\n")
 	if e.LatestSTH != nil {
 		fmt.Fprintf(text, "Latest known log size = %d\n", e.LatestSTH.TreeSize)
 	} else {
 		fmt.Fprintf(text, "Latest known log size = none\n")
 	}
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
 	return text.String()
 }
 func (e *BacklogInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.GetMonitoringURL())
 	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "For details, enable -verbose and see certspotter's stderr output.\n")
-	fmt.Fprintf(text, "\n")
 	fmt.Fprintf(text, "Current log size = %d (as of %s)\n", e.LatestSTH.TreeSize, e.LatestSTH.StoredAt)
 	fmt.Fprintf(text, "Current position = %d\n", e.Position)
 	fmt.Fprintf(text, "         Backlog = %d\n", e.Backlog())
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
 	return text.String()
 }
 func (e *StaleLogListInfo) Text() string {
 	text := new(strings.Builder)
 	fmt.Fprintf(text, "certspotter has been unable to retrieve the log list from %s since %s.\n", e.Source, e.LastSuccess)
 	fmt.Fprintf(text, "\n")
-	fmt.Fprintf(text, "Last error (at %s): %s\n", e.LastErrorTime, e.LastError)
-	fmt.Fprintf(text, "\n")
 	fmt.Fprintf(text, "Consequentially, certspotter may not be monitoring all logs, and might fail to detect certificates.\n")
+	if e.RecentErrors != "" {
+		fmt.Fprintf(text, "\n")
+		fmt.Fprintf(text, "Recent errors (see %s for complete records):\n", e.ErrorsDir)
+		fmt.Fprintf(text, "\n")
+		fmt.Fprint(text, e.RecentErrors)
+	}
 	return text.String()
 }
 

monitor/monitor.go

@@ -206,7 +206,7 @@ func newLogClient(config *Config, ctlog *loglist.Log) (ctclient.Log, ctclient.Is
 				logGetter: client,
 			}, nil
 	default:
-		return nil, nil, fmt.Errorf("log uses unknown protocol")
+		return nil, nil, errors.New("log uses unknown protocol")
 	}
 }
 
@@ -337,8 +337,8 @@ type batch struct {
 	entries      []ctclient.Entry // in range [begin,end)
 }
 
-// create a batch starting from begin, based on sths (which must be non-empty, sorted by TreeSize, and contain only STHs with TreeSize >= begin)
-func newBatch(number uint64, begin uint64, sths []*StoredSTH, downloadJobSize uint64) *batch {
+// Create a batch starting from begin, based on sths (which must be non-empty, sorted by TreeSize, and contain only STHs with TreeSize >= begin).  Returns the batch, plus the remaining STHs.
+func newBatch(number uint64, begin uint64, sths []*StoredSTH, downloadJobSize uint64) (*batch, []*StoredSTH) {
 	batch := &batch{
 		number:       number,
 		begin:        begin,
@@ -357,10 +357,12 @@ func newBatch(number uint64, begin uint64, sths []*StoredSTH, downloadJobSize ui
 			break
 		}
 	}
-	return batch
+	return batch, sths[len(batch.sths):]
 }
 
-func appendSTH(sths []*StoredSTH, sth *StoredSTH) []*StoredSTH {
+// insert sth into sths, which is sorted by TreeSize, and return a new, still-sorted slice.
+// if an equivalent STH is already in sths, it is returned unchanged.
+func insertSTH(sths []*StoredSTH, sth *StoredSTH) []*StoredSTH {
 	i := len(sths)
 	for i > 0 {
 		if sths[i-1].Same(&sth.SignedTreeHead) {
@@ -377,11 +379,11 @@ func appendSTH(sths []*StoredSTH, sth *StoredSTH) []*StoredSTH {
 func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, sthsIn <-chan *cttypes.SignedTreeHead, batchesOut chan<- *batch) error {
 	downloadJobSize := downloadJobSize(ctlog)
 
-	// sths is sorted by TreeSize and contains only STHs with TreeSize >= position
 	sths, err := config.State.LoadSTHs(ctx, ctlog.LogID)
 	if err != nil {
 		return fmt.Errorf("error loading STHs: %w", err)
 	}
+	// sths is sorted by TreeSize but may contain STHs with TreeSize < position; get rid of them
 	for len(sths) > 0 && sths[0].TreeSize < position {
 		// TODO-4: audit sths[0] against log's verified STH
 		if err := config.State.RemoveSTH(ctx, ctlog.LogID, &sths[0].SignedTreeHead); err != nil {
@@ -389,6 +391,7 @@ func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.L
 		}
 		sths = sths[1:]
 	}
+	// from this point, sths is sorted by TreeSize and contains only STHs with TreeSize >= position
 	handleSTH := func(sth *cttypes.SignedTreeHead) error {
 		if sth.TreeSize < position {
 			// TODO-4: audit against log's verified STH
@@ -397,7 +400,7 @@ func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.L
 			if err != nil {
 				return fmt.Errorf("error storing STH: %w", err)
 			}
-			sths = appendSTH(sths, storedSTH)
+			sths = insertSTH(sths, storedSTH)
 		}
 		return nil
 	}
@@ -415,7 +418,7 @@ func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.L
 			}
 		}
 
-		batch := newBatch(number, position, sths, downloadJobSize)
+		batch, remainingSTHs := newBatch(number, position, sths, downloadJobSize)
 
 		if ctlog.IsStaticCTAPI() && batch.end%downloadJobSize != 0 {
 			// Wait to download this partial tile until it's old enough
@@ -443,7 +446,7 @@ func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.L
 		case batchesOut <- batch:
 			number = batch.number + 1
 			position = batch.end
-			sths = sths[len(batch.sths):]
+			sths = remainingSTHs
 		}
 	}
 }
@@ -504,12 +507,11 @@ func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, st
 		if batch.begin != state.DownloadPosition.Size() {
 			panic(fmt.Errorf("saveStateWorker: expected batch to start at %d but got %d instead", state.DownloadPosition.Size(), batch.begin))
 		}
-		rootHash := state.DownloadPosition.CalculateRoot()
 		for {
 			for len(batch.sths) > 0 && batch.sths[0].TreeSize == state.DownloadPosition.Size() {
 				sth := batch.sths[0]
 				batch.sths = batch.sths[1:]
-				if sth.RootHash != rootHash {
+				if rootHash := state.DownloadPosition.CalculateRoot(); sth.RootHash != rootHash {
 					return &verifyEntriesError{
 						sth:             &sth.SignedTreeHead,
 						entriesRootHash: rootHash,
@@ -536,7 +538,6 @@ func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, st
 			batch.entries = batch.entries[1:]
 			leafHash := merkletree.HashLeaf(entry.LeafInput())
 			state.DownloadPosition.Add(leafHash)
-			rootHash = state.DownloadPosition.CalculateRoot()
 		}
 
 		if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil {

monitor/notify.go

@@ -89,18 +89,26 @@ func sendEmail(ctx context.Context, to []string, notif *notification) error {
 	args = append(args, "--")
 	args = append(args, to...)
 
-	sendmail := exec.CommandContext(ctx, sendmailPath(), args...)
+	sendmailCtx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Minute))
+	defer cancel()
+	sendmail := exec.CommandContext(sendmailCtx, sendmailPath(), args...)
 	sendmail.Stdin = stdin
 	sendmail.Stderr = stderr
+	sendmail.WaitDelay = 5 * time.Second
 
-	if err := sendmail.Run(); err == nil {
+	if err := sendmail.Run(); err == nil || err == exec.ErrWaitDelay {
 		return nil
+	} else if sendmailCtx.Err() != nil && ctx.Err() == nil {
+		return fmt.Errorf("error sending email to %v: sendmail command timed out", to)
 	} else if ctx.Err() != nil {
+		// if the context was canceled, we can't be sure that the error is the fault of sendmail, so ignore it
 		return ctx.Err()
 	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
 		return fmt.Errorf("error sending email to %v: sendmail failed with exit code %d and error %q", to, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))
+	} else if isExitError {
+		return fmt.Errorf("error sending email to %v: sendmail terminated by signal with error %q", to, strings.TrimSpace(stderr.String()))
 	} else {
-		return fmt.Errorf("error sending email to %v: %w", to, err)
+		return fmt.Errorf("error sending email to %v: error running sendmail command: %w", to, err)
 	}
 }
 
@@ -111,10 +119,12 @@ func execScript(ctx context.Context, scriptName string, notif *notification) err
 	cmd.Env = os.Environ()
 	cmd.Env = append(cmd.Env, notif.environ...)
 	cmd.Stderr = stderr
+	cmd.WaitDelay = 5 * time.Second
 
-	if err := cmd.Run(); err == nil {
+	if err := cmd.Run(); err == nil || err == exec.ErrWaitDelay {
 		return nil
 	} else if ctx.Err() != nil {
+		// if the context was canceled, we can't be sure that the error is the fault of the script, so ignore it
 		return ctx.Err()
 	} else if exitErr, isExitError := err.(*exec.ExitError); isExitError && exitErr.Exited() {
 		return fmt.Errorf("script %q exited with code %d and error %q", scriptName, exitErr.ExitCode(), strings.TrimSpace(stderr.String()))

monitor/process.go

@@ -107,8 +107,15 @@ func processCertificate(ctx context.Context, config *Config, entry *LogEntry, ce
 	}
 
 	chain, chainErr := getChain(ctx)
-	if errors.Is(chainErr, context.Canceled) {
-		return chainErr
+	if chainErr != nil {
+		if ctx.Err() != nil {
+			// Getting chain failed, but it was probably because our context
+			// has been canceled, so just act like we never called getChain.
+			return ctx.Err()
+		}
+		// Although getting the chain failed, we still want to notify
+		// the user about the matching certificate. We'll include chainErr in the
+		// notification so the user knows why the chain is missing or incorrect.
 	}
 
 	cert := &DiscoveredCert{

monitor/state.go

@@ -85,4 +85,7 @@ type StateProvider interface {
 	// not associated with a log.  Note that most errors are transient, and
 	// certspotter will retry the failed operation later.
 	NotifyError(context.Context, *loglist.Log, error) error
+
+	// Retrieve the specified number of most recent errors.
+	GetErrors(context.Context, *loglist.Log, int) (string, error)
 }

monitor/statedir.go

@@ -145,7 +145,7 @@ func prepareStateDir(stateDir string) error {
 		return fmt.Errorf("%s was created by a newer version of certspotter; upgrade to the latest version of certspotter or remove this directory to start from scratch", stateDir)
 	}
 
-	for _, subdir := range []string{"certs", "logs", "healthchecks"} {
+	for _, subdir := range []string{"certs", "logs", "healthchecks", "errors"} {
 		if err := os.Mkdir(filepath.Join(stateDir, subdir), 0777); err != nil && !errors.Is(err, fs.ErrExist) {
 			return err
 		}

staticcheck.conf

@@ -0,0 +1 @@
+checks = ["inherit", "-ST1005", "-S1002"]