From 185445e158d79399a770d0dec43fa27dd40f820f Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Wed, 29 Apr 2020 11:38:26 -0400 Subject: [PATCH] Retrieve log list from certspotter.org at startup instead of embedding in source The list of logs changes far too frequently (with annual shards and operators dropping out of the ecosystem) to continue embedding in the source code. Breaking change: the -logs option now expects a JSON file in the v2 log list format, as documented at and . You can now specify an HTTPS URL to -logs in addition to a file path. Breaking change: the -underwater option has been removed; if you want this behavior then specify https://loglist.certspotter.org/underwater.json as your log list. --- README | 9 +- cmd/common.go | 61 ++++----- cmd/state.go | 14 +- cmd/submitct/main.go | 53 +++++--- logs.go | 313 ------------------------------------------- scanner.go | 3 +- 6 files changed, 80 insertions(+), 373 deletions(-) delete mode 100644 logs.go diff --git a/README b/README index ca899ea..6355634 100644 --- a/README +++ b/README @@ -93,10 +93,11 @@ COMMAND LINE FLAGS -all_time Scan for certificates from all time, not just those logged since the previous run of Cert Spotter. - -logs FILENAME - JSON file containing logs to scan, in the format documented at - . - Default: use the logs trusted by Chromium. + -logs FILENAME_OR_URL + Filename of HTTPS URL of a JSON file containing logs to monitor, in the format + documented at . + Default: https://loglist.certspotter.org/monitor.json which includes the union + of active logs recognized by Chrome and Apple. -state_dir PATH Directory for storing state. Default: ~/.certspotter -verbose diff --git a/cmd/common.go b/cmd/common.go index ce243e7..54626a7 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -11,6 +11,7 @@ package cmd import ( "bytes" + "crypto/x509" "flag" "fmt" "log" @@ -21,13 +22,15 @@ import ( "software.sslmate.com/src/certspotter" "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" ) +const defaultLogList = "https://loglist.certspotter.org/monitor.json" + var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries (advanced)") var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers (advanced)") var script = flag.String("script", "", "Script to execute when a matching certificate is found") -var logsFilename = flag.String("logs", "", "JSON file containing log information") -var underwater = flag.Bool("underwater", false, "Monitor certificates from distrusted CAs instead of trusted CAs") +var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to monitor") var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates") var verbose = flag.Bool("verbose", false, "Be verbose") var startAtEnd = flag.Bool("start_at_end", false, "Start monitoring logs from the end rather than the beginning") @@ -81,18 +84,12 @@ func LogEntry(info *certspotter.EntryInfo) { } } -func loadLogList() ([]certspotter.LogInfo, error) { - if *logsFilename != "" { - var logFileObj certspotter.LogInfoFile - if err := readJSONFile(*logsFilename, &logFileObj); err != nil { - return nil, fmt.Errorf("Error reading logs file: %s: %s", *logsFilename, err) - } - return logFileObj.Logs, nil - } else if *underwater { - return certspotter.UnderwaterLogs, nil - } else { - return certspotter.DefaultLogs, nil +func loadLogList() ([]*loglist.Log, error) { + list, err := loglist.Load(*logsURL) + if err != nil { + return nil, fmt.Errorf("Error loading log list: %s", err) } + return list.AllLogs(), nil } type logHandle struct { @@ -102,14 +99,14 @@ type logHandle struct { verifiedSTH *ct.SignedTreeHead } -func makeLogHandle(logInfo *certspotter.LogInfo) (*logHandle, error) { +func makeLogHandle(logInfo *loglist.Log) (*logHandle, error) { ctlog := new(logHandle) - logKey, err := logInfo.ParsedPublicKey() + logKey, err := x509.ParsePKIXPublicKey(logInfo.Key) if err != nil { return nil, fmt.Errorf("Bad public key: %s", err) } - ctlog.scanner = certspotter.NewScanner(logInfo.FullURI(), logInfo.ID(), logKey, &certspotter.ScannerOptions{ + ctlog.scanner = certspotter.NewScanner(logInfo.URL, logInfo.LogID, logKey, &certspotter.ScannerOptions{ BatchSize: *batchSize, NumWorkers: *numWorkers, Quiet: !*verbose, @@ -134,7 +131,7 @@ func makeLogHandle(logInfo *certspotter.LogInfo) (*logHandle, error) { return nil, fmt.Errorf("Error loading legacy STH: %s", err) } if legacySTH != nil { - log.Print(logInfo.Url, ": Initializing log state from legacy state directory") + log.Print(logInfo.URL, ": Initializing log state from legacy state directory") ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(legacySTH) if err != nil { return nil, fmt.Errorf("Error reconstructing Merkle Tree for legacy STH: %s", err) @@ -243,59 +240,59 @@ func (ctlog *logHandle) scan(processCallback certspotter.ProcessCallback) error return nil } -func processLog(logInfo *certspotter.LogInfo, processCallback certspotter.ProcessCallback) int { +func processLog(logInfo *loglist.Log, processCallback certspotter.ProcessCallback) int { ctlog, err := makeLogHandle(logInfo) if err != nil { - log.Print(logInfo.Url, ": ", err) + log.Print(logInfo.URL, ": ", err) return 1 } if err := ctlog.refresh(); err != nil { - log.Print(logInfo.Url, ": ", err) + log.Print(logInfo.URL, ": ", err) return 1 } if err := ctlog.audit(); err != nil { - log.Print(logInfo.Url, ": ", err) + log.Print(logInfo.URL, ": ", err) return 1 } if *allTime { ctlog.tree = certspotter.EmptyCollapsedMerkleTree() if *verbose { - log.Printf("%s: Scanning all %d entries in the log because -all_time option specified", logInfo.Url, ctlog.verifiedSTH.TreeSize) + log.Printf("%s: Scanning all %d entries in the log because -all_time option specified", logInfo.URL, ctlog.verifiedSTH.TreeSize) } } else if ctlog.tree != nil { if *verbose { - log.Printf("%s: Existing log; scanning %d new entries since previous scan", logInfo.Url, ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize()) + log.Printf("%s: Existing log; scanning %d new entries since previous scan", logInfo.URL, ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize()) } } else if *startAtEnd { ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(ctlog.verifiedSTH) if err != nil { - log.Print("%s: Error reconstructing Merkle Tree: %s", logInfo.Url, err) + log.Print("%s: Error reconstructing Merkle Tree: %s", logInfo.URL, err) return 1 } if *verbose { - log.Printf("%s: New log; not scanning %d existing entries because -start_at_end option was specified", logInfo.Url, ctlog.verifiedSTH.TreeSize) + log.Printf("%s: New log; not scanning %d existing entries because -start_at_end option was specified", logInfo.URL, ctlog.verifiedSTH.TreeSize) } } else { ctlog.tree = certspotter.EmptyCollapsedMerkleTree() if *verbose { - log.Printf("%s: New log; scanning all %d entries in the log (use the -start_at_end option to scan new logs from the end rather than the beginning)", logInfo.Url, ctlog.verifiedSTH.TreeSize) + log.Printf("%s: New log; scanning all %d entries in the log (use the -start_at_end option to scan new logs from the end rather than the beginning)", logInfo.URL, ctlog.verifiedSTH.TreeSize) } } if err := ctlog.state.StoreTree(ctlog.tree); err != nil { - log.Printf("%s: Error storing tree: %s\n", logInfo.Url, err) + log.Printf("%s: Error storing tree: %s\n", logInfo.URL, err) return 1 } if err := ctlog.scan(processCallback); err != nil { - log.Print(logInfo.Url, ": ", err) + log.Print(logInfo.URL, ": ", err) return 1 } if *verbose { - log.Printf("%s: Final log size = %d, final root hash = %x", logInfo.Url, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash) + log.Printf("%s: Final log size = %d, final root hash = %x", logInfo.URL, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash) } return 0 @@ -330,10 +327,10 @@ func Main(statePath string, processCallback certspotter.ProcessCallback) int { } processLogResults := make(chan int) - for i := range logs { - go func(logInfo *certspotter.LogInfo) { + for _, logInfo := range logs { + go func(logInfo *loglist.Log) { processLogResults <- processLog(logInfo, processCallback) - }(&logs[i]) + }(logInfo) } exitCode := 0 diff --git a/cmd/state.go b/cmd/state.go index 6ae4df3..4b33243 100644 --- a/cmd/state.go +++ b/cmd/state.go @@ -21,16 +21,16 @@ import ( "strconv" "strings" - "software.sslmate.com/src/certspotter" "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/loglist" ) type State struct { path string } -func legacySTHFilename(logInfo *certspotter.LogInfo) string { - return strings.Replace(strings.Replace(logInfo.FullURI(), "://", "_", 1), "/", "_", -1) +func legacySTHFilename(logInfo *loglist.Log) string { + return strings.Replace(strings.Replace(logInfo.URL, "://", "_", 1), "/", "_", -1) } func readVersionFile(statePath string) (int, error) { @@ -161,11 +161,11 @@ func (state *State) SaveCert(isPrecert bool, certs [][]byte) (bool, string, erro return false, path, nil } -func (state *State) OpenLogState(logInfo *certspotter.LogInfo) (*LogState, error) { - return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.ID()))) +func (state *State) OpenLogState(logInfo *loglist.Log) (*LogState, error) { + return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.LogID))) } -func (state *State) GetLegacySTH(logInfo *certspotter.LogInfo) (*ct.SignedTreeHead, error) { +func (state *State) GetLegacySTH(logInfo *loglist.Log) (*ct.SignedTreeHead, error) { sth, err := readSTHFile(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo))) if err != nil { if os.IsNotExist(err) { @@ -176,7 +176,7 @@ func (state *State) GetLegacySTH(logInfo *certspotter.LogInfo) (*ct.SignedTreeHe } return sth, nil } -func (state *State) RemoveLegacySTH(logInfo *certspotter.LogInfo) error { +func (state *State) RemoveLegacySTH(logInfo *loglist.Log) error { err := os.Remove(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo))) os.Remove(filepath.Join(state.path, "legacy_sths")) return err diff --git a/cmd/submitct/main.go b/cmd/submitct/main.go index ad2a8e5..6bddd17 100644 --- a/cmd/submitct/main.go +++ b/cmd/submitct/main.go @@ -13,26 +13,33 @@ import ( "software.sslmate.com/src/certspotter" "software.sslmate.com/src/certspotter/ct" "software.sslmate.com/src/certspotter/ct/client" + "software.sslmate.com/src/certspotter/loglist" "bytes" "crypto/sha256" + "crypto/x509" "encoding/pem" "flag" "fmt" "io/ioutil" "log" "os" + "strings" "sync" "sync/atomic" "time" ) +const defaultLogList = "https://loglist.certspotter.org/submit.json" + var verbose = flag.Bool("v", false, "Enable verbose output") +var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to submit to") type Certificate struct { Subject []byte Issuer []byte Raw []byte + Expiration time.Time } func (cert *Certificate) Fingerprint() [32]byte { @@ -62,10 +69,16 @@ func parseCertificate(data []byte) (*Certificate, error) { return nil, err } + validity, err := tbs.ParseValidity() + if err != nil { + return nil, err + } + return &Certificate{ Subject: tbs.Subject.FullBytes, Issuer: tbs.Issuer.FullBytes, Raw: data, + Expiration: validity.NotAfter, }, nil } @@ -101,19 +114,19 @@ func (certs *CertificateBunch) FindBySubject(subject []byte) *Certificate { } type Log struct { - info certspotter.LogInfo - verify *ct.SignatureVerifier - client *client.LogClient + *loglist.Log + *ct.SignatureVerifier + *client.LogClient } func (ctlog *Log) SubmitChain(chain Chain) (*ct.SignedCertificateTimestamp, error) { rawCerts := chain.GetRawCerts() - sct, err := ctlog.client.AddChain(rawCerts) + sct, err := ctlog.AddChain(rawCerts) if err != nil { return nil, err } - if err := certspotter.VerifyX509SCT(sct, rawCerts[0], ctlog.verify); err != nil { + if err := certspotter.VerifyX509SCT(sct, rawCerts[0], ctlog.SignatureVerifier); err != nil { return nil, fmt.Errorf("Bad SCT signature: %s", err) } return sct, nil @@ -137,20 +150,25 @@ func main() { log.Fatalf("Error reading stdin: %s", err) } - logs := make([]Log, 0, len(certspotter.OpenLogs)) - for _, loginfo := range certspotter.OpenLogs { - pubkey, err := loginfo.ParsedPublicKey() + list, err := loglist.Load(*logsURL) + if err != nil { + log.Fatalf("Error loading log list: %s", err) + } + + var logs []Log + for _, ctlog := range list.AllLogs() { + pubkey, err := x509.ParsePKIXPublicKey(ctlog.Key) if err != nil { - log.Fatalf("%s: Failed to parse log public key: %s", loginfo.Url, err) + log.Fatalf("%s: Failed to parse log public key: %s", ctlog.URL, err) } - verify, err := ct.NewSignatureVerifier(pubkey) + verifier, err := ct.NewSignatureVerifier(pubkey) if err != nil { - log.Fatalf("%s: Failed to create signature verifier for log: %s", loginfo.Url, err) + log.Fatalf("%s: Failed to create signature verifier for log: %s", ctlog.URL, err) } logs = append(logs, Log{ - info: loginfo, - verify: verify, - client: client.New(loginfo.FullURI()), + Log: ctlog, + SignatureVerifier: verifier, + LogClient: client.New(strings.TrimRight(ctlog.URL, "/")), }) } @@ -186,15 +204,18 @@ func main() { continue } for _, ctlog := range logs { + if !ctlog.AcceptsExpiration(chain[0].Expiration) { + continue + } wg.Add(1) go func(fingerprint [32]byte, ctlog Log) { sct, err := ctlog.SubmitChain(chain) if err != nil { - log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.info.Url, err) + log.Printf("%x (%s): %s: Submission Error: %s", fingerprint, cn, ctlog.URL, err) atomic.AddUint32(&submitErrors, 1) } else if *verbose { timestamp := time.Unix(int64(sct.Timestamp)/1000, int64(sct.Timestamp%1000)*1000000) - log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.info.Url, timestamp) + log.Printf("%x (%s): %s: Submitted at %s", fingerprint, cn, ctlog.URL, timestamp) } wg.Done() }(fingerprint, ctlog) diff --git a/logs.go b/logs.go deleted file mode 100644 index 2e37ee9..0000000 --- a/logs.go +++ /dev/null @@ -1,313 +0,0 @@ -// Copyright (C) 2016 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package certspotter - -import ( - "crypto" - "crypto/sha256" - "crypto/x509" - "encoding/base64" - "flag" - "time" -) - -var http_flag = flag.Bool("http", false, "Connect to CT logs over http instead of https, useful for testing") - -type LogInfoFile struct { - Logs []LogInfo `json:"logs"` -} -type LogInfo struct { - Description string `json:"description"` - Key []byte `json:"key"` - Url string `json:"url"` - MMD int `json:"maximum_merge_delay"` - CertExpiryBegin *time.Time `json:"cert_expiry_begin"` - CertExpiryEnd *time.Time `json:"cert_expiry_end"` -} - -func (info *LogInfo) FullURI() string { - if *http_flag { - return "http://" + info.Url - } - return "https://" + info.Url -} - -func (info *LogInfo) ParsedPublicKey() (crypto.PublicKey, error) { - if info.Key != nil { - return x509.ParsePKIXPublicKey(info.Key) - } else { - return nil, nil - } -} - -func (info *LogInfo) ID() []byte { - sum := sha256.Sum256(info.Key) - return sum[:] -} - -var DefaultLogs = []LogInfo{ - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="), - Url: "ct.googleapis.com/pilot", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE1/TMabLkDpCjiupacAlP7xNi0I1JYP8bQFAHDG1xhtolSY1l4QgNRzRrvSe8liE+NPWHdjGxfx3JhTsN9x8/6Q=="), - Url: "ct.googleapis.com/aviator", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEAkbFvhu7gkAW6MHSrBlpE1n4+HCFRkC5OLAjgqhkTH+/uzSfSl8ois8ZxAD2NgaTZe1M9akhYlrYkes4JECs6A=="), - Url: "ct1.digicert-ct.com/log", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg=="), - Url: "ct.googleapis.com/rocketeer", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAv7UIYZopMgTTJWPp2IXhhuAf1l6a9zM7gBvntj5fLaFm9pVKhKYhVnno94XuXeN8EsDgiSIJIj66FpUGvai5samyetZhLocRuXhAiXXbDNyQ4KR51tVebtEq2zT0mT9liTtGwiksFQccyUsaVPhsHq9gJ2IKZdWauVA2Fm5x9h8B9xKn/L/2IaMpkIYtd967TNTP/dLPgixN1PLCLaypvurDGSVDsuWabA3FHKWL9z8wr7kBkbdpEhLlg2H+NAC+9nGKx+tQkuhZ/hWR65aX+CNUPy2OB9/u2rNPyDydb988LENXoUcMkQT0dU3aiYGkFAY0uZjD2vH97TM20xYtNQIDAQAB"), - Url: "ctserver.cnnic.cn", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAETtK8v7MICve56qTHHDhhBOuV4IlUaESxZryCfk9QbG9co/CqPvTsgPDbCpp6oFtyAHwlDhnvr7JijXRD9Cb2FA=="), - Url: "ct.googleapis.com/icarus", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEmyGDvYXsRJsNyXSrYc9DjHsIa2xzb4UR7ZxVoV6mrc9iZB7xjI6+NrOiwH+P/xxkRmOFG6Jel20q37hTh58rA=="), - Url: "ct.googleapis.com/skydiver", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEjicnerZVCXTrbEuUhGW85BXx6lrYfA43zro/bAna5ymW00VQb94etBzSg4j/KS/Oqf/fNN51D8DMGA2ULvw3AQ=="), - Url: "ctlog-gen2.api.venafi.com", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE7+R9dC4VFbbpuyOL+yy14ceAmEf7QGlo/EmtYU6DRzwat43f/3swtLr/L8ugFOOt1YU/RFmMjGCL17ixv66MZw=="), - Url: "mammoth.ct.comodo.com", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE8m/SiQ8/xfiHHqtls9m7FyOMBg4JVZY9CgiixXGz0akvKD6DEL8S0ERmFe9U4ZiA0M4kbT5nmuk3I85Sk4bagA=="), - Url: "sabre.ct.comodo.com", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEI3MQm+HzXvaYa2mVlhB4zknbtAT8cSxakmBoJcBKGqGwYS0bhxSpuvABM1kdBTDpQhXnVdcq+LSiukXJRpGHVg=="), - Url: "ct.googleapis.com/logs/argon2019", - MMD: 86400, - CertExpiryBegin: makeTime(1546300800), - CertExpiryEnd: makeTime(1577836800), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6Tx2p1yKY4015NyIYvdrk36es0uAc1zA4PQ+TGRY+3ZjUTIYY9Wyu+3q/147JG4vNVKLtDWarZwVqGkg6lAYzA=="), - Url: "ct.googleapis.com/logs/argon2020", - MMD: 86400, - CertExpiryBegin: makeTime(1577836800), - CertExpiryEnd: makeTime(1609459200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAETeBmZOrzZKo4xYktx9gI2chEce3cw/tbr5xkoQlmhB18aKfsxD+MnILgGNl0FOm0eYGilFVi85wLRIOhK8lxKw=="), - Url: "ct.googleapis.com/logs/argon2021", - MMD: 86400, - CertExpiryBegin: makeTime(1609459200), - CertExpiryEnd: makeTime(1640995200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEzF05L2a4TH/BLgOhNKPoioYCrkoRxvcmajeb8Dj4XQmNY+gxa4Zmz3mzJTwe33i0qMVp+rfwgnliQ/bM/oFmhA=="), - Url: "ct2.digicert-ct.com/log", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEkZHz1v5r8a9LmXSMegYZAg4UW+Ug56GtNfJTDNFZuubEJYgWf4FcC5D+ZkYwttXTDSo4OkanG9b3AI4swIQ28g=="), - Url: "ct.cloudflare.com/logs/nimbus2019", - MMD: 86400, - CertExpiryBegin: makeTime(1546300800), - CertExpiryEnd: makeTime(1577836800), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE01EAhx4o0zPQrXTcYjgCt4MVFsT0Pwjzb1RwrM0lhWDlxAYPP6/gyMCXNkOn/7KFsjL7rwk78tHMpY8rXn8AYg=="), - Url: "ct.cloudflare.com/logs/nimbus2020", - MMD: 86400, - CertExpiryBegin: makeTime(1577836800), - CertExpiryEnd: makeTime(1609459200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAExpon7ipsqehIeU1bmpog9TFo4Pk8+9oN8OYHl1Q2JGVXnkVFnuuvPgSo2Ep+6vLffNLcmEbxOucz03sFiematg=="), - Url: "ct.cloudflare.com/logs/nimbus2021", - MMD: 86400, - CertExpiryBegin: makeTime(1609459200), - CertExpiryEnd: makeTime(1640995200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAESLJHTlAycmJKDQxIv60pZG8g33lSYxYpCi5gteI6HLevWbFVCdtZx+m9b+0LrwWWl/87mkNN6xE0M4rnrIPA/w=="), - Url: "ct.cloudflare.com/logs/nimbus2022", - MMD: 86400, - CertExpiryBegin: makeTime(1640995200), - CertExpiryEnd: makeTime(1672531200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEi/8tkhjLRp0SXrlZdTzNkTd6HqmcmXiDJz3fAdWLgOhjmv4mohvRhwXul9bgW0ODgRwC9UGAgH/vpGHPvIS1qA=="), - Url: "ct.cloudflare.com/logs/nimbus2023", - MMD: 86400, - CertExpiryBegin: makeTime(1672531200), - CertExpiryEnd: makeTime(1704067200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEkZd/ow8X+FSVWAVSf8xzkFohcPph/x6pS1JHh7g1wnCZ5y/8Hk6jzJxs6t3YMAWz2CPd4VkCdxwKexGhcFxD9A=="), - Url: "yeti2019.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1546300800), - CertExpiryEnd: makeTime(1577836800), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEURAG+Zo0ac3n37ifZKUhBFEV6jfcCzGIRz3tsq8Ca9BP/5XUHy6ZiqsPaAEbVM0uI3Tm9U24RVBHR9JxDElPmg=="), - Url: "yeti2020.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1577836800), - CertExpiryEnd: makeTime(1609459200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE6J4EbcpIAl1+AkSRsbhoY5oRTj3VoFfaf1DlQkfi7Rbe/HcjfVtrwN8jaC+tQDGjF+dqvKhWJAQ6Q6ev6q9Mew=="), - Url: "yeti2021.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1609459200), - CertExpiryEnd: makeTime(1640995200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEn/jYHd77W1G1+131td5mEbCdX/1v/KiYW5hPLcOROvv+xA8Nw2BDjB7y+RGyutD2vKXStp/5XIeiffzUfdYTJg=="), - Url: "yeti2022.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1640995200), - CertExpiryEnd: makeTime(1672531200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE4hHIyMVIrR9oShgbQMYEk8WX1lmkfFKB448Gn93KbsZnnwljDHY6MQqEnWfKGgMOq0gh3QK48c5ZB3UKSIFZ4g=="), - Url: "nessie2020.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1577836800), - CertExpiryEnd: makeTime(1609459200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE9o7AiwrbGBIX6Lnc47I6OfLMdZnRzKoP5u072nBi6vpIOEooktTi1gNwlRPzGC2ySGfuc1xLDeaA/wSFGgpYFg=="), - Url: "nessie2021.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1609459200), - CertExpiryEnd: makeTime(1640995200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJyTdaAMoy/5jvg4RR019F2ihEV1McclBKMe2okuX7MCv/C87v+nxsfz1Af+p+0lADGMkmNd5LqZVqxbGvlHYcQ=="), - Url: "nessie2022.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1640995200), - CertExpiryEnd: makeTime(1672531200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEXu8iQwSCRSf2CbITGpUpBtFVt8+I0IU0d1C36Lfe1+fbwdaI0Z5FktfM2fBoI1bXBd18k2ggKGYGgdZBgLKTg=="), - Url: "nessie2023.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1672531200), - CertExpiryEnd: makeTime(1704067200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfQ0DsdWYitzwFTvG3F4Nbj8Nv5XIVYzQpkyWsU4nuSYlmcwrAp6m092fsdXEw6w1BAeHlzaqrSgNfyvZaJ9y0Q=="), - Url: "yeti2023.ct.digicert.com/log", - MMD: 86400, - CertExpiryBegin: makeTime(1672531200), - CertExpiryEnd: makeTime(1704067200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEeIPc6fGmuBg6AJkv/z7NFckmHvf/OqmjchZJ6wm2qN200keRDg352dWpi7CHnSV51BpQYAj1CQY5JuRAwrrDwg=="), - Url: "ct.googleapis.com/logs/argon2022", - MMD: 86400, - CertExpiryBegin: makeTime(1640995200), - CertExpiryEnd: makeTime(1672531200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAE0JCPZFJOQqyEti5M8j13ALN3CAVHqkVM4yyOcKWCu2yye5yYeqDpEXYoALIgtM3TmHtNlifmt+4iatGwLpF3eA=="), - Url: "ct.googleapis.com/logs/argon2023", - MMD: 86400, - CertExpiryBegin: makeTime(1672531200), - CertExpiryEnd: makeTime(1704067200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEchY+C+/vzj5g3ZXLY3q5qY1Kb2zcYYCmRV4vg6yU84WI0KV00HuO/8XuQqLwLZPjwtCymeLhQunSxgAnaXSuzg=="), - Url: "ct.googleapis.com/logs/xenon2023", - MMD: 86400, - CertExpiryBegin: makeTime(1672531200), - CertExpiryEnd: makeTime(1704067200), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfzb42Zdr/h7hgqgDCo1vrNJqGqbcUvJGJEER9DDqp19W/wFSB0l166hD+U5cAXchpH8ZkBNUuvOHS0OnJ4oJrQ=="), - Url: "oak.ct.letsencrypt.org/2020", - MMD: 86400, - CertExpiryBegin: makeTime(1577836800), - CertExpiryEnd: makeTime(1609977600), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAELsYzGMNwo8rBIlaklBIdmD2Ofn6HkfrjK0Ukz1uOIUC6Lm0jTITCXhoIdjs7JkyXnwuwYiJYiH7sE1YeKu8k9w=="), - Url: "oak.ct.letsencrypt.org/2021", - MMD: 86400, - CertExpiryBegin: makeTime(1609459200), - CertExpiryEnd: makeTime(1641513600), - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEhjyxDVIjWt5u9sB/o2S8rcGJ2pdZTGA8+IpXhI/tvKBjElGE5r3de4yAfeOPhqTqqc+o7vPgXnDgu/a9/B+RLg=="), - Url: "oak.ct.letsencrypt.org/2022", - MMD: 86400, - CertExpiryBegin: makeTime(1640995200), - CertExpiryEnd: makeTime(1673049600), - }, -} - -// Logs which monitor certs from distrusted roots -var UnderwaterLogs = []LogInfo{ - { - Description: "Google 'Submariner' log", - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEOfifIGLUV1Voou9JLfA5LZreRLSUMOCeeic8q3Dw0fpRkGMWV0Gtq20fgHQweQJeLVmEByQj9p81uIW4QkWkTw=="), - Url: "ct.googleapis.com/submariner", - MMD: 86400, - }, -} - -// Logs which accept submissions from anyone -var OpenLogs = []LogInfo{ - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEfahLEimAoz2t01p3uMziiLOl/fHTDM0YDOhBRuiBARsV4UvxG2LdNgoIGLrtCzWE0J5APC2em4JlvR8EEEFMoA=="), - Url: "ct.googleapis.com/pilot", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEIFsYyDzBi7MxCAC/oJBXK7dHjG+1aLCOkHjpoHPqTyghLpzA9BYbqvnV16mAw04vUjyYASVGJCUoI3ctBcJAeg=="), - Url: "ct.googleapis.com/rocketeer", - MMD: 86400, - }, - { - Key: mustDecodeBase64("MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAELPXCMfVjQ2oWSgrewu4fIW4Sfh3lco90CwKZ061pvAI1eflh6c8ACE90pKM0muBDHCN+j0HV7scco4KKQPqq4A=="), - Url: "dodo.ct.comodo.com", - MMD: 86400, - }, -} - -func mustDecodeBase64(str string) []byte { - bytes, err := base64.StdEncoding.DecodeString(str) - if err != nil { - panic("MustDecodeBase64: " + err.Error()) - } - return bytes -} -func makeTime(seconds int64) *time.Time { - t := time.Unix(seconds, 0) - return &t -} diff --git a/scanner.go b/scanner.go index 93f3c66..59295b0 100644 --- a/scanner.go +++ b/scanner.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log" + "strings" "sync" "sync/atomic" "time" @@ -315,7 +316,7 @@ func NewScanner(logUri string, logId []byte, publicKey crypto.PublicKey, opts *S scanner.LogUri = logUri scanner.LogId = logId scanner.publicKey = publicKey - scanner.logClient = client.New(logUri) + scanner.logClient = client.New(strings.TrimRight(logUri, "/")) scanner.opts = *opts return &scanner }