Retrieve log list from certspotter.org at startup instead of embedding in source

The list of logs changes far too frequently (with annual shards and operators
dropping out of the ecosystem) to continue embedding in the source code.

Breaking change: the -logs option now expects a
JSON file in the v2 log list format, as documented at
<https://www.certificate-transparency.org/known-logs> and
<https://www.gstatic.com/ct/log_list/v2/log_list_schema.json>.

You can now specify an HTTPS URL to -logs in addition to a file path.

Breaking change: the -underwater option has been removed; if you want
this behavior then specify https://loglist.certspotter.org/underwater.json
as your log list.

README

@@ -93,10 +93,11 @@ COMMAND LINE FLAGS
   -all_time
 	Scan for certificates from all time, not just those logged since
 	the previous run of Cert Spotter.
-  -logs FILENAME
+  -logs FILENAME_OR_URL
-	JSON file containing logs to scan, in the format documented at
+	Filename of HTTPS URL of a JSON file containing logs to monitor, in the format
-	<https://www.certificate-transparency.org/known-logs>.
+	documented at <https://www.certificate-transparency.org/known-logs>.
-	Default: use the logs trusted by Chromium.
+	Default: https://loglist.certspotter.org/monitor.json which includes the union
+	of active logs recognized by Chrome and Apple.
   -state_dir PATH
 	Directory for storing state. Default: ~/.certspotter
   -verbose

cmd/common.go

@@ -11,6 +11,7 @@ package cmd
 
 import (
 	"bytes"
+	"crypto/x509"
 	"flag"
 	"fmt"
 	"log"
@@ -21,13 +22,15 @@ import (
 
 	"software.sslmate.com/src/certspotter"
 	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
 )
 
+const defaultLogList = "https://loglist.certspotter.org/monitor.json"
+
 var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries (advanced)")
 var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers (advanced)")
 var script = flag.String("script", "", "Script to execute when a matching certificate is found")
-var logsFilename = flag.String("logs", "", "JSON file containing log information")
+var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to monitor")
-var underwater = flag.Bool("underwater", false, "Monitor certificates from distrusted CAs instead of trusted CAs")
 var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates")
 var verbose = flag.Bool("verbose", false, "Be verbose")
 var startAtEnd = flag.Bool("start_at_end", false, "Start monitoring logs from the end rather than the beginning")
@@ -81,18 +84,12 @@ func LogEntry(info *certspotter.EntryInfo) {
 	}
 }
 
-func loadLogList() ([]certspotter.LogInfo, error) {
+func loadLogList() ([]*loglist.Log, error) {
-	if *logsFilename != "" {
+	list, err := loglist.Load(*logsURL)
-		var logFileObj certspotter.LogInfoFile
+	if err != nil {
-		if err := readJSONFile(*logsFilename, &logFileObj); err != nil {
+		return nil, fmt.Errorf("Error loading log list: %s", err)
-			return nil, fmt.Errorf("Error reading logs file: %s: %s", *logsFilename, err)
-		}
-		return logFileObj.Logs, nil
-	} else if *underwater {
-		return certspotter.UnderwaterLogs, nil
-	} else {
-		return certspotter.DefaultLogs, nil
 	}
+	return list.AllLogs(), nil
 }
 
 type logHandle struct {
@@ -102,14 +99,14 @@ type logHandle struct {
 	verifiedSTH *ct.SignedTreeHead
 }
 
-func makeLogHandle(logInfo *certspotter.LogInfo) (*logHandle, error) {
+func makeLogHandle(logInfo *loglist.Log) (*logHandle, error) {
 	ctlog := new(logHandle)
 
-	logKey, err := logInfo.ParsedPublicKey()
+	logKey, err := x509.ParsePKIXPublicKey(logInfo.Key)
 	if err != nil {
 		return nil, fmt.Errorf("Bad public key: %s", err)
 	}
-	ctlog.scanner = certspotter.NewScanner(logInfo.FullURI(), logInfo.ID(), logKey, &certspotter.ScannerOptions{
+	ctlog.scanner = certspotter.NewScanner(logInfo.URL, logInfo.LogID, logKey, &certspotter.ScannerOptions{
 		BatchSize:  *batchSize,
 		NumWorkers: *numWorkers,
 		Quiet:      !*verbose,
@@ -134,7 +131,7 @@ func makeLogHandle(logInfo *certspotter.LogInfo) (*logHandle, error) {
 			return nil, fmt.Errorf("Error loading legacy STH: %s", err)
 		}
 		if legacySTH != nil {
-			log.Print(logInfo.Url, ": Initializing log state from legacy state directory")
+			log.Print(logInfo.URL, ": Initializing log state from legacy state directory")
 			ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(legacySTH)
 			if err != nil {
 				return nil, fmt.Errorf("Error reconstructing Merkle Tree for legacy STH: %s", err)
@@ -243,59 +240,59 @@ func (ctlog *logHandle) scan(processCallback certspotter.ProcessCallback) error
 	return nil
 }
 
-func processLog(logInfo *certspotter.LogInfo, processCallback certspotter.ProcessCallback) int {
+func processLog(logInfo *loglist.Log, processCallback certspotter.ProcessCallback) int {
 	ctlog, err := makeLogHandle(logInfo)
 	if err != nil {
-		log.Print(logInfo.Url, ": ", err)
+		log.Print(logInfo.URL, ": ", err)
 		return 1
 	}
 
 	if err := ctlog.refresh(); err != nil {
-		log.Print(logInfo.Url, ": ", err)
+		log.Print(logInfo.URL, ": ", err)
 		return 1
 	}
 
 	if err := ctlog.audit(); err != nil {
-		log.Print(logInfo.Url, ": ", err)
+		log.Print(logInfo.URL, ": ", err)
 		return 1
 	}
 
 	if *allTime {
 		ctlog.tree = certspotter.EmptyCollapsedMerkleTree()
 		if *verbose {
-			log.Printf("%s: Scanning all %d entries in the log because -all_time option specified", logInfo.Url, ctlog.verifiedSTH.TreeSize)
+			log.Printf("%s: Scanning all %d entries in the log because -all_time option specified", logInfo.URL, ctlog.verifiedSTH.TreeSize)
 		}
 	} else if ctlog.tree != nil {
 		if *verbose {
-			log.Printf("%s: Existing log; scanning %d new entries since previous scan", logInfo.Url, ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize())
+			log.Printf("%s: Existing log; scanning %d new entries since previous scan", logInfo.URL, ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize())
 		}
 	} else if *startAtEnd {
 		ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(ctlog.verifiedSTH)
 		if err != nil {
-			log.Print("%s: Error reconstructing Merkle Tree: %s", logInfo.Url, err)
+			log.Print("%s: Error reconstructing Merkle Tree: %s", logInfo.URL, err)
 			return 1
 		}
 		if *verbose {
-			log.Printf("%s: New log; not scanning %d existing entries because -start_at_end option was specified", logInfo.Url, ctlog.verifiedSTH.TreeSize)
+			log.Printf("%s: New log; not scanning %d existing entries because -start_at_end option was specified", logInfo.URL, ctlog.verifiedSTH.TreeSize)
 		}
 	} else {
 		ctlog.tree = certspotter.EmptyCollapsedMerkleTree()
 		if *verbose {
-			log.Printf("%s: New log; scanning all %d entries in the log (use the -start_at_end option to scan new logs from the end rather than the beginning)", logInfo.Url, ctlog.verifiedSTH.TreeSize)
+			log.Printf("%s: New log; scanning all %d entries in the log (use the -start_at_end option to scan new logs from the end rather than the beginning)", logInfo.URL, ctlog.verifiedSTH.TreeSize)
 		}
 	}
 	if err := ctlog.state.StoreTree(ctlog.tree); err != nil {
-		log.Printf("%s: Error storing tree: %s\n", logInfo.Url, err)
+		log.Printf("%s: Error storing tree: %s\n", logInfo.URL, err)
 		return 1
 	}
 
 	if err := ctlog.scan(processCallback); err != nil {
-		log.Print(logInfo.Url, ": ", err)
+		log.Print(logInfo.URL, ": ", err)
 		return 1
 	}
 
 	if *verbose {
-		log.Printf("%s: Final log size = %d, final root hash = %x", logInfo.Url, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash)
+		log.Printf("%s: Final log size = %d, final root hash = %x", logInfo.URL, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash)
 	}
 
 	return 0
@@ -330,10 +327,10 @@ func Main(statePath string, processCallback certspotter.ProcessCallback) int {
 	}
 
 	processLogResults := make(chan int)
-	for i := range logs {
+	for _, logInfo := range logs {
-		go func(logInfo *certspotter.LogInfo) {
+		go func(logInfo *loglist.Log) {
 			processLogResults <- processLog(logInfo, processCallback)
-		}(&logs[i])
+		}(logInfo)
 	}
 
 	exitCode := 0

cmd/state.go

@@ -21,16 +21,16 @@ import (
 	"strconv"
 	"strings"
 
-	"software.sslmate.com/src/certspotter"
 	"software.sslmate.com/src/certspotter/ct"
+	"software.sslmate.com/src/certspotter/loglist"
 )
 
 type State struct {
 	path string
 }
 
-func legacySTHFilename(logInfo *certspotter.LogInfo) string {
+func legacySTHFilename(logInfo *loglist.Log) string {
-	return strings.Replace(strings.Replace(logInfo.FullURI(), "://", "_", 1), "/", "_", -1)
+	return strings.Replace(strings.Replace(logInfo.URL, "://", "_", 1), "/", "_", -1)
 }
 
 func readVersionFile(statePath string) (int, error) {
@@ -161,11 +161,11 @@ func (state *State) SaveCert(isPrecert bool, certs [][]byte) (bool, string, erro
 	return false, path, nil
 }
 
-func (state *State) OpenLogState(logInfo *certspotter.LogInfo) (*LogState, error) {
+func (state *State) OpenLogState(logInfo *loglist.Log) (*LogState, error) {
-	return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.ID())))
+	return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.LogID)))
 }
 
-func (state *State) GetLegacySTH(logInfo *certspotter.LogInfo) (*ct.SignedTreeHead, error) {
+func (state *State) GetLegacySTH(logInfo *loglist.Log) (*ct.SignedTreeHead, error) {
 	sth, err := readSTHFile(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo)))
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -176,7 +176,7 @@ func (state *State) GetLegacySTH(logInfo *certspotter.LogInfo) (*ct.SignedTreeHe
 	}
 	return sth, nil
 }
-func (state *State) RemoveLegacySTH(logInfo *certspotter.LogInfo) error {
+func (state *State) RemoveLegacySTH(logInfo *loglist.Log) error {
 	err := os.Remove(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo)))
 	os.Remove(filepath.Join(state.path, "legacy_sths"))
 	return err

cmd/submitct/main.go

@@ -13,26 +13,33 @@ import (
 	"software.sslmate.com/src/certspotter"
 	"software.sslmate.com/src/certspotter/ct"
 	"software.sslmate.com/src/certspotter/ct/client"
+	"software.sslmate.com/src/certspotter/loglist"
 
 	"bytes"
 	"crypto/sha256"
+	"crypto/x509"
 	"encoding/pem"
 	"flag"
 	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 )
 
+const defaultLogList = "https://loglist.certspotter.org/submit.json"
+
 var verbose = flag.Bool("v", false, "Enable verbose output")
+var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to submit to")
 
 type Certificate struct {
 	Subject []byte
 	Issuer  []byte
 	Raw     []byte
+	Expiration time.Time
 }
 
 func (cert *Certificate) Fingerprint() [32]byte {
@@ -62,10 +69,16 @@ func parseCertificate(data []byte) (*Certificate, error) {
 		return nil, err
 	}
 
+	validity, err := tbs.ParseValidity()
+	if err != nil {
+		return nil, err
+	}
+
 	return &Certificate{
 		Subject: tbs.Subject.FullBytes,
 		Issuer:  tbs.Issuer.FullBytes,
 		Raw:     data,
+		Expiration: validity.NotAfter,
 	}, nil
 }
 
@@ -101,19 +114,19 @@ func (certs *CertificateBunch) FindBySubject(subject []byte) *Certificate {
 }
 
 type Log struct {
-	info   certspotter.LogInfo
+	*loglist.Log
-	verify *ct.SignatureVerifier
+	*ct.SignatureVerifier
-	client *client.LogClient
+	*client.LogClient
 }
 
 func (ctlog *Log) SubmitChain(chain Chain) (*ct.SignedCertificateTimestamp, error) {
 	rawCerts := chain.GetRawCerts()
-	sct, err := ctlog.client.AddChain(rawCerts)
+	sct, err := ctlog.AddChain(rawCerts)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := certspotter.VerifyX509SCT(sct, rawCerts[0], ctlog.verify); err != nil {
+	if err := certspotter.VerifyX509SCT(sct, rawCerts[0], ctlog.SignatureVerifier); err != nil {
 		return nil, fmt.Errorf("Bad SCT signature: %s", err)
 	}
 	return sct, nil
@@ -137,20 +150,25 @@ func main() {
 		log.Fatalf("Error reading stdin: %s", err)
 	}
 
-	logs := make([]Log, 0, len(certspotter.OpenLogs))
+	list, err := loglist.Load(*logsURL)
-	for _, loginfo := range certspotter.OpenLogs {
-		pubkey, err := loginfo.ParsedPublicKey()
 	if err != nil {
-			log.Fatalf("%s: Failed to parse log public key: %s", loginfo.Url, err)
+		log.Fatalf("Error loading log list: %s", err)
 	}
-		verify, err := ct.NewSignatureVerifier(pubkey)
+
+	var logs []Log
+	for _, ctlog := range list.AllLogs() {
+		pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key)
 		if err != nil {
-			log.Fatalf("%s: Failed to create signature verifier for log: %s", loginfo.Url, err)
+			log.Fatalf("%s: Failed to parse log public key: %s", ctlog.URL, err)
+		}
+		verifier, err := ct.NewSignatureVerifier(pubkey)
+		if err != nil {
+			log.Fatalf("%s: Failed to create signature verifier for log: %s", ctlog.URL, err)
 		}
 		logs = append(logs, Log{
-			info:   loginfo,
+			Log: ctlog,
-			verify: verify,
+			SignatureVerifier: verifier,
-			client: client.New(loginfo.FullURI()),
+			LogClient: client.New(strings.TrimRight(ctlog.URL, "/")),
 		})
 	}
 
@@ -186,15 +204,18 @@ func main() {
 			continue
 		}
 		for _, ctlog := range logs {
+			if !ctlog.AcceptsExpiration(chain[0].Expiration) {
+				continue
+			}
 			wg.Add(1)
 			go func(fingerprint [32]byte, ctlog Log) {
 				sct, err := ctlog.SubmitChain(chain)
 				if err != nil {
-					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.info.Url, err)
+					log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.URL, err)
 					atomic.AddUint32(&submitErrors, 1)
 				} else if *verbose {
 					timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000)
-					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.info.Url, timestamp)
+					log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.URL, timestamp)
 				}
 				wg.Done()
 			}(fingerprint, ctlog)

logs.go

@@ -1,313 +0,0 @@
-// Copyright (C) 2016 Opsmate, Inc.
-//
-// This Source Code Form is subject to the terms of the Mozilla
-// Public License, v. 2.0. If a copy of the MPL was not distributed
-// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
-//
-// This software is distributed WITHOUT A WARRANTY OF ANY KIND.
-// See the Mozilla Public License for details.
-
-package certspotter
-
-import (
-	"crypto"
-	"crypto/sha256"
-	"crypto/x509"
-	"encoding/base64"
-	"flag"
-	"time"
-)
-
-var http_flag = flag.Bool("http", false, "Connect to CT logs over http instead of https, useful for testing")
-
-type LogInfoFile struct {
-	Logs []LogInfo `json:"logs"`
-}
-type LogInfo struct {
-	Description     string     `json:"description"`
-	Key             []byte     `json:"key"`
-	Url             string     `json:"url"`
-	MMD             int        `json:"maximum_merge_delay"`
-	CertExpiryBegin *time.Time `json:"cert_expiry_begin"`
-	CertExpiryEnd   *time.Time `json:"cert_expiry_end"`
-}
-
-func (info *LogInfo) FullURI() string {
-	if *http_flag {
-		return "http://" + info.Url
-	}
-	return "https://" + info.Url
-}
-
-func (info *LogInfo) ParsedPublicKey() (crypto.PublicKey, error) {
-	if info.Key != nil {
-		return x509.ParsePKIXPublicKey(info.Key)
-	} else {
-		return nil, nil
-	}
-}
-
-func (info *LogInfo) ID() []byte {
-	sum := sha256.Sum256(info.Key)
-	return sum[:]
-}
-
-var DefaultLogs = []LogInfo{
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="),
-		Url: "ct.googleapis.com/pilot",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1/TMabLkDpCjiupacAlP7xNi0I1JYP8bQFAHDG1xhtolSY1l4QgNRzRrvSe8liE+NPWHdjGxfx3JhTsN9x8/6Q=="),
-		Url: "ct.googleapis.com/aviator",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="),
-		Url: "ct1.digicert-ct.com/log",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg=="),
-		Url: "ct.googleapis.com/rocketeer",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv7UIYZopMgTTJWPp2IXhhuAf1l6a9zM7gBvntj5fLaFm9pVKhKYhVnno94XuXeN8EsDgiSIJIj66FpUGvai5samyetZhLocRuXhAiXXbDNyQ4KR51tVebtEq2zT0mT9liTtGwiksFQccyUsaVPhsHq9gJ2IKZdWauVA2Fm5x9h8B9xKn/L/2IaMpkIYtd967TNTP/dLPgixN1PLCLaypvurDGSVDsuWabA3FHKWL9z8wr7kBkbdpEhLlg2H+NAC+9nGKx+tQkuhZ/hWR65aX+CNUPy2OB9/u2rNPyDydb988LENXoUcMkQT0dU3aiYGkFAY0uZjD2vH97TM20xYtNQIDAQAB"),
-		Url: "ctserver.cnnic.cn",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAETtK8v7MICve56qTHHDhhBOuV4IlUaESxZryCfk9QbG9co/CqPvTsgPDbCpp6oFtyAHwlDhnvr7JijXRD9Cb2FA=="),
-		Url: "ct.googleapis.com/icarus",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEmyGDvYXsRJsNyXSrYc9DjHsIa2xzb4UR7ZxVoV6mrc9iZB7xjI6+NrOiwH+P/xxkRmOFG6Jel20q37hTh58rA=="),
-		Url: "ct.googleapis.com/skydiver",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEjicnerZVCXTrbEuUhGW85BXx6lrYfA43zro/bAna5ymW00VQb94etBzSg4j/KS/Oqf/fNN51D8DMGA2ULvw3AQ=="),
-		Url: "ctlog-gen2.api.venafi.com",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE7+R9dC4VFbbpuyOL+yy14ceAmEf7QGlo/EmtYU6DRzwat43f/3swtLr/L8ugFOOt1YU/RFmMjGCL17ixv66MZw=="),
-		Url: "mammoth.ct.comodo.com",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8m/SiQ8/xfiHHqtls9m7FyOMBg4JVZY9CgiixXGz0akvKD6DEL8S0ERmFe9U4ZiA0M4kbT5nmuk3I85Sk4bagA=="),
-		Url: "sabre.ct.comodo.com",
-		MMD: 86400,
-	},
-	{
-		Key:             mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEI3MQm+HzXvaYa2mVlhB4zknbtAT8cSxakmBoJcBKGqGwYS0bhxSpuvABM1kdBTDpQhXnVdcq+LSiukXJRpGHVg=="),
-		Url:             "ct.googleapis.com/logs/argon2019",
-		MMD:             86400,
-		CertExpiryBegin: makeTime(1546300800),
-		CertExpiryEnd:   makeTime(1577836800),
-	},
-	{
-		Key:             mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6Tx2p1yKY4015NyIYvdrk36es0uAc1zA4PQ+TGRY+3ZjUTIYY9Wyu+3q/147JG4vNVKLtDWarZwVqGkg6lAYzA=="),
-		Url:             "ct.googleapis.com/logs/argon2020",
-		MMD:             86400,
-		CertExpiryBegin: makeTime(1577836800),
-		CertExpiryEnd:   makeTime(1609459200),
-	},
-	{
-		Key:             mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAETeBmZOrzZKo4xYktx9gI2chEce3cw/tbr5xkoQlmhB18aKfsxD+MnILgGNl0FOm0eYGilFVi85wLRIOhK8lxKw=="),
-		Url:             "ct.googleapis.com/logs/argon2021",
-		MMD:             86400,
-		CertExpiryBegin: makeTime(1609459200),
-		CertExpiryEnd:   makeTime(1640995200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzF05L2a4TH/BLgOhNKPoioYCrkoRxvcmajeb8Dj4XQmNY+gxa4Zmz3mzJTwe33i0qMVp+rfwgnliQ/bM/oFmhA=="),
-		Url: "ct2.digicert-ct.com/log",
-		MMD: 86400,
-	},
-	{
-		Key:             mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEkZHz1v5r8a9LmXSMegYZAg4UW+Ug56GtNfJTDNFZuubEJYgWf4FcC5D+ZkYwttXTDSo4OkanG9b3AI4swIQ28g=="),
-		Url:             "ct.cloudflare.com/logs/nimbus2019",
-		MMD:             86400,
-		CertExpiryBegin: makeTime(1546300800),
-		CertExpiryEnd:   makeTime(1577836800),
-	},
-	{
-		Key:             mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE01EAhx4o0zPQrXTcYjgCt4MVFsT0Pwjzb1RwrM0lhWDlxAYPP6/gyMCXNkOn/7KFsjL7rwk78tHMpY8rXn8AYg=="),
-		Url:             "ct.cloudflare.com/logs/nimbus2020",
-		MMD:             86400,
-		CertExpiryBegin: makeTime(1577836800),
-		CertExpiryEnd:   makeTime(1609459200),
-	},
-	{
-		Key:             mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAExpon7ipsqehIeU1bmpog9TFo4Pk8+9oN8OYHl1Q2JGVXnkVFnuuvPgSo2Ep+6vLffNLcmEbxOucz03sFiematg=="),
-		Url:             "ct.cloudflare.com/logs/nimbus2021",
-		MMD:             86400,
-		CertExpiryBegin: makeTime(1609459200),
-		CertExpiryEnd:   makeTime(1640995200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAESLJHTlAycmJKDQxIv60pZG8g33lSYxYpCi5gteI6HLevWbFVCdtZx+m9b+0LrwWWl/87mkNN6xE0M4rnrIPA/w=="),
-		Url: "ct.cloudflare.com/logs/nimbus2022",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1640995200),
-		CertExpiryEnd: makeTime(1672531200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEi/8tkhjLRp0SXrlZdTzNkTd6HqmcmXiDJz3fAdWLgOhjmv4mohvRhwXul9bgW0ODgRwC9UGAgH/vpGHPvIS1qA=="),
-		Url: "ct.cloudflare.com/logs/nimbus2023",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1672531200),
-		CertExpiryEnd: makeTime(1704067200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEkZd/ow8X+FSVWAVSf8xzkFohcPph/x6pS1JHh7g1wnCZ5y/8Hk6jzJxs6t3YMAWz2CPd4VkCdxwKexGhcFxD9A=="),
-		Url: "yeti2019.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1546300800),
-		CertExpiryEnd: makeTime(1577836800),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEURAG+Zo0ac3n37ifZKUhBFEV6jfcCzGIRz3tsq8Ca9BP/5XUHy6ZiqsPaAEbVM0uI3Tm9U24RVBHR9JxDElPmg=="),
-		Url: "yeti2020.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1577836800),
-		CertExpiryEnd: makeTime(1609459200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6J4EbcpIAl1+AkSRsbhoY5oRTj3VoFfaf1DlQkfi7Rbe/HcjfVtrwN8jaC+tQDGjF+dqvKhWJAQ6Q6ev6q9Mew=="),
-		Url: "yeti2021.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1609459200),
-		CertExpiryEnd: makeTime(1640995200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEn/jYHd77W1G1+131td5mEbCdX/1v/KiYW5hPLcOROvv+xA8Nw2BDjB7y+RGyutD2vKXStp/5XIeiffzUfdYTJg=="),
-		Url: "yeti2022.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1640995200),
-		CertExpiryEnd: makeTime(1672531200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4hHIyMVIrR9oShgbQMYEk8WX1lmkfFKB448Gn93KbsZnnwljDHY6MQqEnWfKGgMOq0gh3QK48c5ZB3UKSIFZ4g=="),
-		Url: "nessie2020.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1577836800),
-		CertExpiryEnd: makeTime(1609459200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9o7AiwrbGBIX6Lnc47I6OfLMdZnRzKoP5u072nBi6vpIOEooktTi1gNwlRPzGC2ySGfuc1xLDeaA/wSFGgpYFg=="),
-		Url: "nessie2021.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1609459200),
-		CertExpiryEnd: makeTime(1640995200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJyTdaAMoy/5jvg4RR019F2ihEV1McclBKMe2okuX7MCv/C87v+nxsfz1Af+p+0lADGMkmNd5LqZVqxbGvlHYcQ=="),
-		Url: "nessie2022.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1640995200),
-		CertExpiryEnd: makeTime(1672531200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEXu8iQwSCRSf2CbITGpUpBtFVt8+I0IU0d1C36Lfe1+fbwdaI0Z5FktfM2fBoI1bXBd18k2ggKGYGgdZBgLKTg=="),
-		Url: "nessie2023.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1672531200),
-		CertExpiryEnd: makeTime(1704067200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfQ0DsdWYitzwFTvG3F4Nbj8Nv5XIVYzQpkyWsU4nuSYlmcwrAp6m092fsdXEw6w1BAeHlzaqrSgNfyvZaJ9y0Q=="),
-		Url: "yeti2023.ct.digicert.com/log",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1672531200),
-		CertExpiryEnd: makeTime(1704067200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEeIPc6fGmuBg6AJkv/z7NFckmHvf/OqmjchZJ6wm2qN200keRDg352dWpi7CHnSV51BpQYAj1CQY5JuRAwrrDwg=="),
-		Url: "ct.googleapis.com/logs/argon2022",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1640995200),
-		CertExpiryEnd: makeTime(1672531200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE0JCPZFJOQqyEti5M8j13ALN3CAVHqkVM4yyOcKWCu2yye5yYeqDpEXYoALIgtM3TmHtNlifmt+4iatGwLpF3eA=="),
-		Url: "ct.googleapis.com/logs/argon2023",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1672531200),
-		CertExpiryEnd: makeTime(1704067200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEchY+C+/vzj5g3ZXLY3q5qY1Kb2zcYYCmRV4vg6yU84WI0KV00HuO/8XuQqLwLZPjwtCymeLhQunSxgAnaXSuzg=="),
-		Url: "ct.googleapis.com/logs/xenon2023",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1672531200),
-		CertExpiryEnd: makeTime(1704067200),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfzb42Zdr/h7hgqgDCo1vrNJqGqbcUvJGJEER9DDqp19W/wFSB0l166hD+U5cAXchpH8ZkBNUuvOHS0OnJ4oJrQ=="),
-		Url: "oak.ct.letsencrypt.org/2020",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1577836800),
-		CertExpiryEnd: makeTime(1609977600),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAELsYzGMNwo8rBIlaklBIdmD2Ofn6HkfrjK0Ukz1uOIUC6Lm0jTITCXhoIdjs7JkyXnwuwYiJYiH7sE1YeKu8k9w=="),
-		Url: "oak.ct.letsencrypt.org/2021",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1609459200),
-		CertExpiryEnd: makeTime(1641513600),
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEhjyxDVIjWt5u9sB/o2S8rcGJ2pdZTGA8+IpXhI/tvKBjElGE5r3de4yAfeOPhqTqqc+o7vPgXnDgu/a9/B+RLg=="),
-		Url: "oak.ct.letsencrypt.org/2022",
-		MMD: 86400,
-		CertExpiryBegin: makeTime(1640995200),
-		CertExpiryEnd: makeTime(1673049600),
-	},
-}
-
-// Logs which monitor certs from distrusted roots
-var UnderwaterLogs = []LogInfo{
-	{
-		Description: "Google 'Submariner' log",
-		Key:         mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEOfifIGLUV1Voou9JLfA5LZreRLSUMOCeeic8q3Dw0fpRkGMWV0Gtq20fgHQweQJeLVmEByQj9p81uIW4QkWkTw=="),
-		Url:         "ct.googleapis.com/submariner",
-		MMD:         86400,
-	},
-}
-
-// Logs which accept submissions from anyone
-var OpenLogs = []LogInfo{
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="),
-		Url: "ct.googleapis.com/pilot",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg=="),
-		Url: "ct.googleapis.com/rocketeer",
-		MMD: 86400,
-	},
-	{
-		Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAELPXCMfVjQ2oWSgrewu4fIW4Sfh3lco90CwKZ061pvAI1eflh6c8ACE90pKM0muBDHCN+j0HV7scco4KKQPqq4A=="),
-		Url: "dodo.ct.comodo.com",
-		MMD: 86400,
-	},
-}
-
-func mustDecodeBase64(str string) []byte {
-	bytes, err := base64.StdEncoding.DecodeString(str)
-	if err != nil {
-		panic("MustDecodeBase64: " + err.Error())
-	}
-	return bytes
-}
-func makeTime(seconds int64) *time.Time {
-	t := time.Unix(seconds, 0)
-	return &t
-}

scanner.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"log"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -315,7 +316,7 @@ func NewScanner(logUri string, logId []byte, publicKey crypto.PublicKey, opts *S
 	scanner.LogUri = logUri
 	scanner.LogId = logId
 	scanner.publicKey = publicKey
-	scanner.logClient = client.New(logUri)
+	scanner.logClient = client.New(strings.TrimRight(logUri, "/"))
 	scanner.opts = *opts
 	return &scanner
 }