From 29ed939006d7fb12b1d2f5069a5f59aa26c258e1 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Fri, 3 Feb 2023 14:47:47 -0500 Subject: [PATCH] Remove old code --- auditing.go | 213 ---------------------------- cmd/common.go | 358 ----------------------------------------------- cmd/helpers.go | 87 ------------ cmd/log_state.go | 145 ------------------- cmd/state.go | 220 ----------------------------- helpers.go | 247 -------------------------------- scanner.go | 299 --------------------------------------- version.go | 12 -- 8 files changed, 1581 deletions(-) delete mode 100644 auditing.go delete mode 100644 cmd/common.go delete mode 100644 cmd/helpers.go delete mode 100644 cmd/log_state.go delete mode 100644 cmd/state.go delete mode 100644 scanner.go delete mode 100644 version.go diff --git a/auditing.go b/auditing.go deleted file mode 100644 index da6680f..0000000 --- a/auditing.go +++ /dev/null @@ -1,213 +0,0 @@ -// Copyright (C) 2016 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package certspotter - -import ( - "bytes" - "crypto/sha256" - "encoding/json" - "errors" - "software.sslmate.com/src/certspotter/ct" -) - -func reverseHashes(hashes []ct.MerkleTreeNode) { - for i := 0; i < len(hashes)/2; i++ { - j := len(hashes) - i - 1 - hashes[i], hashes[j] = hashes[j], hashes[i] - } -} - -func VerifyConsistencyProof(proof ct.ConsistencyProof, first *ct.SignedTreeHead, second *ct.SignedTreeHead) bool { - // TODO: make sure every hash in proof is right length? otherwise input to hashChildren is ambiguous - if second.TreeSize < first.TreeSize { - // Can't be consistent if tree got smaller - return false - } - if first.TreeSize == second.TreeSize { - if !(bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]) && len(proof) == 0) { - return false - } - return true - } - if first.TreeSize == 0 { - // The purpose of the consistency proof is to ensure the append-only - // nature of the tree; i.e. that the first tree is a "prefix" of the - // second tree. If the first tree is empty, then it's trivially a prefix - // of the second tree, so no proof is needed. - if len(proof) != 0 { - return false - } - return true - } - // Guaranteed that 0 < first.TreeSize < second.TreeSize - - node := first.TreeSize - 1 - lastNode := second.TreeSize - 1 - - // While we're the right child, everything is in both trees, so move one level up. - for node%2 == 1 { - node /= 2 - lastNode /= 2 - } - - var newHash ct.MerkleTreeNode - var oldHash ct.MerkleTreeNode - if node > 0 { - if len(proof) == 0 { - return false - } - newHash = proof[0] - proof = proof[1:] - } else { - // The old tree was balanced, so we already know the first hash to use - newHash = first.SHA256RootHash[:] - } - oldHash = newHash - - for node > 0 { - if node%2 == 1 { - // node is a right child; left sibling exists in both trees - if len(proof) == 0 { - return false - } - newHash = hashChildren(proof[0], newHash) - oldHash = hashChildren(proof[0], oldHash) - proof = proof[1:] - } else if node < lastNode { - // node is a left child; rigth sibling only exists in the new tree - if len(proof) == 0 { - return false - } - newHash = hashChildren(newHash, proof[0]) - proof = proof[1:] - } // else node == lastNode: node is a left child with no sibling in either tree - node /= 2 - lastNode /= 2 - } - - if !bytes.Equal(oldHash, first.SHA256RootHash[:]) { - return false - } - - // If trees have different height, continue up the path to reach the new root - for lastNode > 0 { - if len(proof) == 0 { - return false - } - newHash = hashChildren(newHash, proof[0]) - proof = proof[1:] - lastNode /= 2 - } - - if !bytes.Equal(newHash, second.SHA256RootHash[:]) { - return false - } - - return true -} - -func hashNothing() ct.MerkleTreeNode { - return sha256.New().Sum(nil) -} - -func hashLeaf(leafBytes []byte) ct.MerkleTreeNode { - hasher := sha256.New() - hasher.Write([]byte{0x00}) - hasher.Write(leafBytes) - return hasher.Sum(nil) -} - -func hashChildren(left ct.MerkleTreeNode, right ct.MerkleTreeNode) ct.MerkleTreeNode { - hasher := sha256.New() - hasher.Write([]byte{0x01}) - hasher.Write(left) - hasher.Write(right) - return hasher.Sum(nil) -} - -type CollapsedMerkleTree struct { - nodes []ct.MerkleTreeNode - size uint64 -} - -func calculateNumNodes(size uint64) int { - numNodes := 0 - for size > 0 { - numNodes += int(size & 1) - size >>= 1 - } - return numNodes -} -func EmptyCollapsedMerkleTree() *CollapsedMerkleTree { - return &CollapsedMerkleTree{} -} -func NewCollapsedMerkleTree(nodes []ct.MerkleTreeNode, size uint64) (*CollapsedMerkleTree, error) { - if len(nodes) != calculateNumNodes(size) { - return nil, errors.New("NewCollapsedMerkleTree: nodes has incorrect size") - } - return &CollapsedMerkleTree{nodes: nodes, size: size}, nil -} -func CloneCollapsedMerkleTree(source *CollapsedMerkleTree) *CollapsedMerkleTree { - nodes := make([]ct.MerkleTreeNode, len(source.nodes)) - copy(nodes, source.nodes) - return &CollapsedMerkleTree{nodes: nodes, size: source.size} -} - -func (tree *CollapsedMerkleTree) Add(hash ct.MerkleTreeNode) { - tree.nodes = append(tree.nodes, hash) - tree.size++ - size := tree.size - for size%2 == 0 { - left, right := tree.nodes[len(tree.nodes)-2], tree.nodes[len(tree.nodes)-1] - tree.nodes = tree.nodes[:len(tree.nodes)-2] - tree.nodes = append(tree.nodes, hashChildren(left, right)) - size /= 2 - } -} - -func (tree *CollapsedMerkleTree) CalculateRoot() ct.MerkleTreeNode { - if len(tree.nodes) == 0 { - return hashNothing() - } - i := len(tree.nodes) - 1 - hash := tree.nodes[i] - for i > 0 { - i -= 1 - hash = hashChildren(tree.nodes[i], hash) - } - return hash -} - -func (tree *CollapsedMerkleTree) GetSize() uint64 { - return tree.size -} - -func (tree *CollapsedMerkleTree) MarshalJSON() ([]byte, error) { - return json.Marshal(map[string]interface{}{ - "nodes": tree.nodes, - "size": tree.size, - }) -} - -func (tree *CollapsedMerkleTree) UnmarshalJSON(b []byte) error { - var rawTree struct { - Nodes []ct.MerkleTreeNode `json:"nodes"` - Size uint64 `json:"size"` - } - if err := json.Unmarshal(b, &rawTree); err != nil { - return errors.New("Failed to unmarshal CollapsedMerkleTree: " + err.Error()) - } - if len(rawTree.Nodes) != calculateNumNodes(rawTree.Size) { - return errors.New("Failed to unmarshal CollapsedMerkleTree: nodes has incorrect length") - } - tree.size = rawTree.Size - tree.nodes = rawTree.Nodes - return nil -} diff --git a/cmd/common.go b/cmd/common.go deleted file mode 100644 index 5733e27..0000000 --- a/cmd/common.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright (C) 2016-2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "bytes" - "crypto/x509" - "flag" - "fmt" - "log" - "os" - "path/filepath" - "sync" - - "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/loglist" -) - -const defaultLogList = "https://loglist.certspotter.org/monitor.json" - -var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries (advanced)") -var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers (advanced)") -var script = flag.String("script", "", "Script to execute when a matching certificate is found") -var logsURL = flag.String("logs", defaultLogList, "File path or URL of JSON list of logs to monitor") -var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates") -var verbose = flag.Bool("verbose", false, "Be verbose") -var showVersion = flag.Bool("version", false, "Print version and exit") -var startAtEnd = flag.Bool("start_at_end", false, "Start monitoring logs from the end rather than the beginning") -var allTime = flag.Bool("all_time", false, "Scan certs from all time, not just since last scan") -var state *State - -var printMutex sync.Mutex - -func homedir() string { - homedir, err := os.UserHomeDir() - if err != nil { - panic(fmt.Errorf("unable to determine home directory: %w", err)) - } - return homedir -} - -func DefaultStateDir(programName string) string { - return filepath.Join(homedir(), "."+programName) -} - -func DefaultConfigDir(programName string) string { - return filepath.Join(homedir(), "."+programName) -} - -func LogEntry(info *certspotter.EntryInfo) { - if !*noSave { - var alreadyPresent bool - var err error - alreadyPresent, info.Filename, err = state.SaveCert(info.IsPrecert, info.FullChain) - if err != nil { - log.Print(err) // important error (system) - } - if alreadyPresent { - return - } - } - - if *script != "" { - if err := info.InvokeHookScript(*script); err != nil { - log.Print(err) // important error (system) - } - } else { - printMutex.Lock() - info.Write(os.Stdout) - fmt.Fprintf(os.Stdout, "\n") - printMutex.Unlock() - } -} - -func loadLogList() ([]*loglist.Log, error) { - list, err := loglist.Load(*logsURL) - if err != nil { - return nil, fmt.Errorf("Error loading log list: %s", err) - } - return list.AllLogs(), nil -} - -type logHandle struct { - scanner *certspotter.Scanner - state *LogState - tree *certspotter.CollapsedMerkleTree - verifiedSTH *ct.SignedTreeHead -} - -func makeLogHandle(logInfo *loglist.Log) (*logHandle, error) { - ctlog := new(logHandle) - - logKey, err := x509.ParsePKIXPublicKey(logInfo.Key) - if err != nil { - return nil, fmt.Errorf("Bad public key: %s", err) - } - ctlog.scanner = certspotter.NewScanner(logInfo.URL, logInfo.LogID, logKey, &certspotter.ScannerOptions{ - BatchSize: *batchSize, - NumWorkers: *numWorkers, - Quiet: !*verbose, - }) - - ctlog.state, err = state.OpenLogState(logInfo) - if err != nil { - return nil, fmt.Errorf("Error opening state directory: %s", err) - } - ctlog.tree, err = ctlog.state.GetTree() - if err != nil { - return nil, fmt.Errorf("Error loading tree: %s", err) - } - ctlog.verifiedSTH, err = ctlog.state.GetVerifiedSTH() - if err != nil { - return nil, fmt.Errorf("Error loading verified STH: %s", err) - } - - if ctlog.tree == nil && ctlog.verifiedSTH == nil { // This branch can be removed eventually - legacySTH, err := state.GetLegacySTH(logInfo) - if err != nil { - return nil, fmt.Errorf("Error loading legacy STH: %s", err) - } - if legacySTH != nil { - log.Print(logInfo.URL, ": Initializing log state from legacy state directory") - ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(legacySTH) - if err != nil { - return nil, fmt.Errorf("Error reconstructing Merkle Tree for legacy STH: %s", err) - } - if err := ctlog.state.StoreTree(ctlog.tree); err != nil { - return nil, fmt.Errorf("Error storing tree: %s", err) - } - if err := ctlog.state.StoreVerifiedSTH(legacySTH); err != nil { - return nil, fmt.Errorf("Error storing verified STH: %s", err) - } - state.RemoveLegacySTH(logInfo) - } - } - - return ctlog, nil -} - -func (ctlog *logHandle) refresh() error { - if *verbose { - log.Print(ctlog.scanner.LogUri, ": Retrieving latest STH from log") - } - latestSTH, err := ctlog.scanner.GetSTH() - if err != nil { - return fmt.Errorf("Error retrieving STH from log: %s", err) - } - if ctlog.verifiedSTH == nil { - if *verbose { - log.Printf("%s: No existing STH is known; presuming latest STH (%d) is valid", ctlog.scanner.LogUri, latestSTH.TreeSize) - } - ctlog.verifiedSTH = latestSTH - if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil { - return fmt.Errorf("Error storing verified STH: %s", err) - } - } else { - if err := ctlog.state.StoreUnverifiedSTH(latestSTH); err != nil { - return fmt.Errorf("Error storing unverified STH: %s", err) - } - } - return nil -} - -func (ctlog *logHandle) verifySTH(sth *ct.SignedTreeHead) error { - isValid, err := ctlog.scanner.CheckConsistency(ctlog.verifiedSTH, sth) - if err != nil { - return fmt.Errorf("Error fetching consistency proof: %s", err) - } - if !isValid { - return fmt.Errorf("Consistency proof between %d and %d is invalid", ctlog.verifiedSTH.TreeSize, sth.TreeSize) - } - return nil -} - -func (ctlog *logHandle) audit() error { - sths, err := ctlog.state.GetUnverifiedSTHs() - if err != nil { - return fmt.Errorf("Error loading unverified STHs: %s", err) - } - - for _, sth := range sths { - if *verbose { - log.Printf("%s: Verifying consistency of STH %d (%x) with previously-verified STH %d (%x)", ctlog.scanner.LogUri, sth.TreeSize, sth.SHA256RootHash, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash) - } - if err := ctlog.verifySTH(sth); err != nil { - log.Printf("%s: Unable to verify consistency of STH %d (%s) (if this error persists, it should be construed as misbehavior by the log): %s", ctlog.scanner.LogUri, sth.TreeSize, ctlog.state.UnverifiedSTHFilename(sth), err) // important error (log) - continue - } - if sth.TreeSize > ctlog.verifiedSTH.TreeSize { - if *verbose { - log.Printf("%s: STH %d (%x) is now the latest verified STH", ctlog.scanner.LogUri, sth.TreeSize, sth.SHA256RootHash) - } - ctlog.verifiedSTH = sth - if err := ctlog.state.StoreVerifiedSTH(ctlog.verifiedSTH); err != nil { - return fmt.Errorf("Error storing verified STH: %s", err) - } - } - if err := ctlog.state.RemoveUnverifiedSTH(sth); err != nil { - return fmt.Errorf("Error removing redundant STH: %s", err) - } - } - - return nil -} - -func (ctlog *logHandle) scan(processCallback certspotter.ProcessCallback) error { - startIndex := int64(ctlog.tree.GetSize()) - endIndex := int64(ctlog.verifiedSTH.TreeSize) - - if endIndex > startIndex { - tree := certspotter.CloneCollapsedMerkleTree(ctlog.tree) - - if err := ctlog.scanner.Scan(startIndex, endIndex, processCallback, tree); err != nil { - return fmt.Errorf("Error scanning log (if this error persists, it should be construed as misbehavior by the log): %s", err) - } - - rootHash := tree.CalculateRoot() - if !bytes.Equal(rootHash, ctlog.verifiedSTH.SHA256RootHash[:]) { - return fmt.Errorf("Log has misbehaved: log entries at tree size %d do not correspond to signed tree root", ctlog.verifiedSTH.TreeSize) - } - - ctlog.tree = tree - if err := ctlog.state.StoreTree(ctlog.tree); err != nil { - return fmt.Errorf("Error storing tree: %s", err) - } - } - - return nil -} - -func processLog(logInfo *loglist.Log, processCallback certspotter.ProcessCallback) int { - ctlog, err := makeLogHandle(logInfo) - if err != nil { - log.Print(logInfo.URL, ": ", err) // important error (system) - return 1 - } - - if err := ctlog.refresh(); err != nil { - log.Print(logInfo.URL, ": ", err) // important error (both system and log) - return 1 - } - - if err := ctlog.audit(); err != nil { - log.Print(logInfo.URL, ": ", err) // important error (system) - return 1 - } - - if *allTime { - ctlog.tree = certspotter.EmptyCollapsedMerkleTree() - if *verbose { - log.Printf("%s: Scanning all %d entries in the log because -all_time option specified", logInfo.URL, ctlog.verifiedSTH.TreeSize) - } - } else if ctlog.tree != nil { - if *verbose { - log.Printf("%s: Existing log; scanning %d new entries since previous scan", logInfo.URL, ctlog.verifiedSTH.TreeSize-ctlog.tree.GetSize()) - } - } else if *startAtEnd { - ctlog.tree, err = ctlog.scanner.MakeCollapsedMerkleTree(ctlog.verifiedSTH) - if err != nil { - log.Printf("%s: Error reconstructing Merkle Tree: %s", logInfo.URL, err) // important error (log) - return 1 - } - if *verbose { - log.Printf("%s: New log; not scanning %d existing entries because -start_at_end option was specified", logInfo.URL, ctlog.verifiedSTH.TreeSize) - } - } else { - ctlog.tree = certspotter.EmptyCollapsedMerkleTree() - if *verbose { - log.Printf("%s: New log; scanning all %d entries in the log (use the -start_at_end option to scan new logs from the end rather than the beginning)", logInfo.URL, ctlog.verifiedSTH.TreeSize) - } - } - if err := ctlog.state.StoreTree(ctlog.tree); err != nil { - log.Printf("%s: Error storing tree: %s\n", logInfo.URL, err) // important error (system) - return 1 - } - - if err := ctlog.scan(processCallback); err != nil { - log.Print(logInfo.URL, ": ", err) // important error (both system and log) - return 1 - } - - if *verbose { - log.Printf("%s: Final log size = %d, final root hash = %x", logInfo.URL, ctlog.verifiedSTH.TreeSize, ctlog.verifiedSTH.SHA256RootHash) - } - - return 0 -} - -func ParseFlags() { - flag.Parse() - if *showVersion { - fmt.Fprintf(os.Stdout, "Cert Spotter %s\n", certspotter.Version) - os.Exit(0) - } -} - -func Main(statePath string, processCallback certspotter.ProcessCallback) int { - var err error - - logs, err := loadLogList() - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) // important error (loglist) - return 1 - } - - state, err = OpenState(statePath) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: %s\n", os.Args[0], err) // important error (system) - return 1 - } - locked, err := state.Lock() - if err != nil { - fmt.Fprintf(os.Stderr, "%s: Error locking state directory: %s\n", os.Args[0], err) // important error (system) - return 1 - } - if !locked { - var otherPidInfo string - if otherPid := state.LockingPid(); otherPid != 0 { - otherPidInfo = fmt.Sprintf(" (as process ID %d)", otherPid) - } - fmt.Fprintf(os.Stderr, "%s: Another instance of %s is already running%s; remove the file %s if this is not the case\n", os.Args[0], os.Args[0], otherPidInfo, state.LockFilename()) // important error (system) - return 1 - } - - processLogResults := make(chan int) - for _, logInfo := range logs { - go func(logInfo *loglist.Log) { - processLogResults <- processLog(logInfo, processCallback) - }(logInfo) - } - - exitCode := 0 - for range logs { - exitCode |= <-processLogResults - } - - if state.IsFirstRun() && exitCode == 0 { - if err := state.WriteOnceFile(); err != nil { - fmt.Fprintf(os.Stderr, "%s: Error writing once file: %s\n", os.Args[0], err) // important error (system) - exitCode |= 1 - } - } - - if err := state.Unlock(); err != nil { - fmt.Fprintf(os.Stderr, "%s: Error unlocking state directory: %s\n", os.Args[0], err) // important error (system) - exitCode |= 1 - } - - return exitCode -} diff --git a/cmd/helpers.go b/cmd/helpers.go deleted file mode 100644 index 7fa4f44..0000000 --- a/cmd/helpers.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright (C) 2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "io/ioutil" - "os" - - "software.sslmate.com/src/certspotter/ct" -) - -func fileExists(path string) bool { - _, err := os.Lstat(path) - return err == nil -} - -func writeFile(filename string, data []byte, perm os.FileMode) error { - tempname := filename + ".new" - if err := ioutil.WriteFile(tempname, data, perm); err != nil { - return err - } - if err := os.Rename(tempname, filename); err != nil { - os.Remove(tempname) - return err - } - return nil -} - -func writeJSONFile(filename string, obj interface{}, perm os.FileMode) error { - tempname := filename + ".new" - f, err := os.OpenFile(tempname, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, perm) - if err != nil { - return err - } - if err := json.NewEncoder(f).Encode(obj); err != nil { - f.Close() - os.Remove(tempname) - return err - } - if err := f.Close(); err != nil { - os.Remove(tempname) - return err - } - if err := os.Rename(tempname, filename); err != nil { - os.Remove(tempname) - return err - } - return nil -} - -func readJSONFile(filename string, obj interface{}) error { - bytes, err := ioutil.ReadFile(filename) - if err != nil { - return err - } - if err = json.Unmarshal(bytes, obj); err != nil { - return err - } - return nil -} - -func readSTHFile(filename string) (*ct.SignedTreeHead, error) { - sth := new(ct.SignedTreeHead) - if err := readJSONFile(filename, sth); err != nil { - return nil, err - } - return sth, nil -} - -func sha256sum(data []byte) []byte { - sum := sha256.Sum256(data) - return sum[:] -} - -func sha256hex(data []byte) string { - return hex.EncodeToString(sha256sum(data)) -} diff --git a/cmd/log_state.go b/cmd/log_state.go deleted file mode 100644 index 209bf96..0000000 --- a/cmd/log_state.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (C) 2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "crypto/sha256" - "encoding/base64" - "encoding/binary" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/ct" -) - -type LogState struct { - path string -} - -// generate a filename that uniquely identifies the STH (within the context of a particular log) -func sthFilename(sth *ct.SignedTreeHead) string { - hasher := sha256.New() - switch sth.Version { - case ct.V1: - binary.Write(hasher, binary.LittleEndian, sth.Timestamp) - binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash) - default: - panic(fmt.Sprintf("Unsupported STH version %d", sth.Version)) - } - // For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic) - return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json" -} - -func makeLogStateDir(logStatePath string) error { - if err := os.Mkdir(logStatePath, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", logStatePath, err) - } - for _, subdir := range []string{"unverified_sths"} { - path := filepath.Join(logStatePath, subdir) - if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", path, err) - } - } - return nil -} - -func OpenLogState(logStatePath string) (*LogState, error) { - if err := makeLogStateDir(logStatePath); err != nil { - return nil, fmt.Errorf("Error creating log state directory: %s", err) - } - return &LogState{path: logStatePath}, nil -} - -func (logState *LogState) VerifiedSTHFilename() string { - return filepath.Join(logState.path, "sth.json") -} - -func (logState *LogState) GetVerifiedSTH() (*ct.SignedTreeHead, error) { - sth, err := readSTHFile(logState.VerifiedSTHFilename()) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } else { - return nil, err - } - } - return sth, nil -} - -func (logState *LogState) StoreVerifiedSTH(sth *ct.SignedTreeHead) error { - return writeJSONFile(logState.VerifiedSTHFilename(), sth, 0666) -} - -func (logState *LogState) GetUnverifiedSTHs() ([]*ct.SignedTreeHead, error) { - dir, err := os.Open(filepath.Join(logState.path, "unverified_sths")) - if err != nil { - if os.IsNotExist(err) { - return []*ct.SignedTreeHead{}, nil - } else { - return nil, err - } - } - filenames, err := dir.Readdirnames(0) - if err != nil { - return nil, err - } - - sths := make([]*ct.SignedTreeHead, 0, len(filenames)) - for _, filename := range filenames { - if !strings.HasPrefix(filename, ".") { - sth, _ := readSTHFile(filepath.Join(dir.Name(), filename)) - if sth != nil { - sths = append(sths, sth) - } - } - } - return sths, nil -} - -func (logState *LogState) UnverifiedSTHFilename(sth *ct.SignedTreeHead) string { - return filepath.Join(logState.path, "unverified_sths", sthFilename(sth)) -} - -func (logState *LogState) StoreUnverifiedSTH(sth *ct.SignedTreeHead) error { - filename := logState.UnverifiedSTHFilename(sth) - if fileExists(filename) { - return nil - } - return writeJSONFile(filename, sth, 0666) -} - -func (logState *LogState) RemoveUnverifiedSTH(sth *ct.SignedTreeHead) error { - filename := logState.UnverifiedSTHFilename(sth) - err := os.Remove(filepath.Join(filename)) - if err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - -func (logState *LogState) GetTree() (*certspotter.CollapsedMerkleTree, error) { - tree := new(certspotter.CollapsedMerkleTree) - if err := readJSONFile(filepath.Join(logState.path, "tree.json"), tree); err != nil { - if os.IsNotExist(err) { - return nil, nil - } else { - return nil, err - } - } - return tree, nil -} - -func (logState *LogState) StoreTree(tree *certspotter.CollapsedMerkleTree) error { - return writeJSONFile(filepath.Join(logState.path, "tree.json"), tree, 0666) -} diff --git a/cmd/state.go b/cmd/state.go deleted file mode 100644 index 5ecd117..0000000 --- a/cmd/state.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright (C) 2017 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package cmd - -import ( - "bytes" - "encoding/base64" - "encoding/pem" - "fmt" - "io/ioutil" - "log" - "os" - "path/filepath" - "strconv" - "strings" - - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/loglist" -) - -type State struct { - path string -} - -func legacySTHFilename(logInfo *loglist.Log) string { - return strings.Replace(strings.Replace(logInfo.URL, "://", "_", 1), "/", "_", -1) -} - -func readVersionFile(statePath string) (int, error) { - versionFilePath := filepath.Join(statePath, "version") - versionBytes, err := ioutil.ReadFile(versionFilePath) - if err == nil { - version, err := strconv.Atoi(string(bytes.TrimSpace(versionBytes))) - if err != nil { - return -1, fmt.Errorf("%s: contains invalid integer: %s", versionFilePath, err) - } - if version < 0 { - return -1, fmt.Errorf("%s: contains negative integer", versionFilePath) - } - return version, nil - } else if os.IsNotExist(err) { - if fileExists(filepath.Join(statePath, "sths")) { - // Original version of certspotter had no version file. - // Infer version 0 if "sths" directory is present. - return 0, nil - } - return -1, nil - } else { - return -1, fmt.Errorf("%s: %s", versionFilePath, err) - } -} - -func writeVersionFile(statePath string) error { - version := 1 - versionString := fmt.Sprintf("%d\n", version) - versionFilePath := filepath.Join(statePath, "version") - if err := ioutil.WriteFile(versionFilePath, []byte(versionString), 0666); err != nil { - return fmt.Errorf("%s: %s\n", versionFilePath, err) - } - return nil -} - -func makeStateDir(statePath string) error { - if err := os.Mkdir(statePath, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", statePath, err) - } - for _, subdir := range []string{"certs", "logs"} { - path := filepath.Join(statePath, subdir) - if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("%s: %s", path, err) - } - } - return nil -} - -func OpenState(statePath string) (*State, error) { - version, err := readVersionFile(statePath) - if err != nil { - return nil, fmt.Errorf("Error reading version file: %s", err) - } - - if version < 1 { - if err := makeStateDir(statePath); err != nil { - return nil, fmt.Errorf("Error creating state directory: %s", err) - } - if version == 0 { - log.Printf("Migrating state directory (%s) to new layout...", statePath) - if err := os.Rename(filepath.Join(statePath, "sths"), filepath.Join(statePath, "legacy_sths")); err != nil { - return nil, fmt.Errorf("Error migrating STHs directory: %s", err) - } - for _, subdir := range []string{"evidence", "legacy_sths"} { - os.Remove(filepath.Join(statePath, subdir)) - } - if err := ioutil.WriteFile(filepath.Join(statePath, "once"), []byte{}, 0666); err != nil { - return nil, fmt.Errorf("Error creating once file: %s", err) - } - } - if err := writeVersionFile(statePath); err != nil { - return nil, fmt.Errorf("Error writing version file: %s", err) - } - } else if version > 1 { - return nil, fmt.Errorf("%s was created by a newer version of Cert Spotter; please remove this directory or upgrade Cert Spotter", statePath) - } - - return &State{path: statePath}, nil -} - -func (state *State) IsFirstRun() bool { - return !fileExists(filepath.Join(state.path, "once")) -} - -func (state *State) WriteOnceFile() error { - if err := ioutil.WriteFile(filepath.Join(state.path, "once"), []byte{}, 0666); err != nil { - return fmt.Errorf("Error writing once file: %s", err) - } - return nil -} - -func (state *State) SaveCert(isPrecert bool, certs [][]byte) (bool, string, error) { - if len(certs) == 0 { - return false, "", fmt.Errorf("Cannot write an empty certificate chain") - } - - fingerprint := sha256hex(certs[0]) - prefixPath := filepath.Join(state.path, "certs", fingerprint[0:2]) - var filenameSuffix string - if isPrecert { - filenameSuffix = ".precert.pem" - } else { - filenameSuffix = ".cert.pem" - } - if err := os.Mkdir(prefixPath, 0777); err != nil && !os.IsExist(err) { - return false, "", fmt.Errorf("Failed to create prefix directory %s: %s", prefixPath, err) - } - path := filepath.Join(prefixPath, fingerprint+filenameSuffix) - file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) - if err != nil { - if os.IsExist(err) { - return true, path, nil - } else { - return false, path, fmt.Errorf("Failed to open %s for writing: %s", path, err) - } - } - for _, cert := range certs { - if err := pem.Encode(file, &pem.Block{Type: "CERTIFICATE", Bytes: cert}); err != nil { - file.Close() - return false, path, fmt.Errorf("Error writing to %s: %s", path, err) - } - } - if err := file.Close(); err != nil { - return false, path, fmt.Errorf("Error writing to %s: %s", path, err) - } - - return false, path, nil -} - -func (state *State) OpenLogState(logInfo *loglist.Log) (*LogState, error) { - return OpenLogState(filepath.Join(state.path, "logs", base64.RawURLEncoding.EncodeToString(logInfo.LogID[:]))) -} - -func (state *State) GetLegacySTH(logInfo *loglist.Log) (*ct.SignedTreeHead, error) { - sth, err := readSTHFile(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo))) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } else { - return nil, err - } - } - return sth, nil -} -func (state *State) RemoveLegacySTH(logInfo *loglist.Log) error { - err := os.Remove(filepath.Join(state.path, "legacy_sths", legacySTHFilename(logInfo))) - os.Remove(filepath.Join(state.path, "legacy_sths")) - return err -} -func (state *State) LockFilename() string { - return filepath.Join(state.path, "lock") -} -func (state *State) Lock() (bool, error) { - file, err := os.OpenFile(state.LockFilename(), os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) - if err != nil { - if os.IsExist(err) { - return false, nil - } else { - return false, err - } - } - if _, err := fmt.Fprintf(file, "%d\n", os.Getpid()); err != nil { - file.Close() - os.Remove(state.LockFilename()) - return false, err - } - if err := file.Close(); err != nil { - os.Remove(state.LockFilename()) - return false, err - } - return true, nil -} -func (state *State) Unlock() error { - return os.Remove(state.LockFilename()) -} -func (state *State) LockingPid() int { - pidBytes, err := ioutil.ReadFile(state.LockFilename()) - if err != nil { - return 0 - } - pid, err := strconv.Atoi(string(bytes.TrimSpace(pidBytes))) - if err != nil { - return 0 - } - return pid -} diff --git a/helpers.go b/helpers.go index 7bf175a..cd524bf 100644 --- a/helpers.go +++ b/helpers.go @@ -10,17 +10,8 @@ package certspotter import ( - "bytes" - "crypto/sha256" - "encoding/hex" "fmt" - "io" "math/big" - "os" - "os/exec" - "strconv" - "strings" - "time" "software.sslmate.com/src/certspotter/ct" ) @@ -29,49 +20,6 @@ func IsPrecert(entry *ct.LogEntry) bool { return entry.Leaf.TimestampedEntry.EntryType == ct.PrecertLogEntryType } -func GetFullChain(entry *ct.LogEntry) [][]byte { - certs := make([][]byte, 0, len(entry.Chain)+1) - - if entry.Leaf.TimestampedEntry.EntryType == ct.X509LogEntryType { - certs = append(certs, entry.Leaf.TimestampedEntry.X509Entry) - } - for _, cert := range entry.Chain { - certs = append(certs, cert) - } - - return certs -} - -func formatSerialNumber(serial *big.Int) string { - if serial != nil { - return fmt.Sprintf("%x", serial) - } else { - return "" - } -} - -func sha256sum(data []byte) []byte { - sum := sha256.Sum256(data) - return sum[:] -} - -func sha256hex(data []byte) string { - return hex.EncodeToString(sha256sum(data)) -} - -type EntryInfo struct { - LogUri string - Entry *ct.LogEntry - IsPrecert bool - FullChain [][]byte // first entry is logged X509 cert or pre-cert - CertInfo *CertInfo - ParseError error // set iff CertInfo is nil - Identifiers *Identifiers - IdentifiersParseError error - Filename string - Bygone bool -} - type CertInfo struct { TBS *TBSCertificate @@ -133,201 +81,6 @@ func MakeCertInfoFromLogEntry(entry *ct.LogEntry) (*CertInfo, error) { } } -func (info *CertInfo) NotBefore() *time.Time { - if info.ValidityParseError == nil { - return &info.Validity.NotBefore - } else { - return nil - } -} - -func (info *CertInfo) NotAfter() *time.Time { - if info.ValidityParseError == nil { - return &info.Validity.NotAfter - } else { - return nil - } -} - -func (info *CertInfo) PubkeyHash() string { - return sha256hex(info.TBS.GetRawPublicKey()) -} - -func (info *CertInfo) PubkeyHashBytes() []byte { - return sha256sum(info.TBS.GetRawPublicKey()) -} - -func (info *CertInfo) Environ() []string { - env := make([]string, 0, 10) - - env = append(env, "PUBKEY_HASH="+info.PubkeyHash()) // deprecated, not documented - env = append(env, "PUBKEY_SHA256="+info.PubkeyHash()) - - if info.SerialNumberParseError != nil { - env = append(env, "SERIAL_PARSE_ERROR="+info.SerialNumberParseError.Error()) - } else { - env = append(env, "SERIAL="+formatSerialNumber(info.SerialNumber)) // generally unsafe to use - } - - if info.ValidityParseError != nil { - env = append(env, "VALIDITY_PARSE_ERROR="+info.ValidityParseError.Error()) - } else { - env = append(env, "NOT_BEFORE="+info.Validity.NotBefore.String()) - env = append(env, "NOT_BEFORE_UNIXTIME="+strconv.FormatInt(info.Validity.NotBefore.Unix(), 10)) - env = append(env, "NOT_AFTER="+info.Validity.NotAfter.String()) - env = append(env, "NOT_AFTER_UNIXTIME="+strconv.FormatInt(info.Validity.NotAfter.Unix(), 10)) - } - - if info.SubjectParseError != nil { - env = append(env, "SUBJECT_PARSE_ERROR="+info.SubjectParseError.Error()) - } else { - env = append(env, "SUBJECT_DN="+info.Subject.String()) - } - - if info.IssuerParseError != nil { - env = append(env, "ISSUER_PARSE_ERROR="+info.IssuerParseError.Error()) - } else { - env = append(env, "ISSUER_DN="+info.Issuer.String()) - } - - return env -} - -func (info *EntryInfo) HasParseErrors() bool { - return info.ParseError != nil || - info.IdentifiersParseError != nil || - info.CertInfo.SubjectParseError != nil || - info.CertInfo.IssuerParseError != nil || - info.CertInfo.SANsParseError != nil || - info.CertInfo.SerialNumberParseError != nil || - info.CertInfo.ValidityParseError != nil || - info.CertInfo.IsCAParseError != nil -} - -func (info *EntryInfo) Fingerprint() string { - if len(info.FullChain) > 0 { - return sha256hex(info.FullChain[0]) - } else { - return "" - } -} - -func (info *EntryInfo) FingerprintBytes() []byte { - if len(info.FullChain) > 0 { - return sha256sum(info.FullChain[0]) - } else { - return []byte{} - } -} - -func (info *EntryInfo) typeString() string { - if info.IsPrecert { - return "precert" - } else { - return "cert" - } -} - -func (info *EntryInfo) typeFriendlyString() string { - if info.IsPrecert { - return "Pre-certificate" - } else { - return "Certificate" - } -} - -func yesnoString(value bool) string { - if value { - return "yes" - } else { - return "no" - } -} - -func (info *EntryInfo) Environ() []string { - env := []string{ - "FINGERPRINT=" + info.Fingerprint(), // deprecated, not documented - "CERT_SHA256=" + info.Fingerprint(), - "CERT_PARSEABLE=" + yesnoString(info.ParseError == nil), // seems redundant with PARSE_ERROR - "LOG_URI=" + info.LogUri, // questionable utility - "ENTRY_INDEX=" + strconv.FormatInt(info.Entry.Index, 10), // questionable utility - } - - if info.Filename != "" { - env = append(env, "CERT_FILENAME="+info.Filename) - } - if info.ParseError != nil { - env = append(env, "PARSE_ERROR="+info.ParseError.Error()) - } else if info.CertInfo != nil { - certEnv := info.CertInfo.Environ() - env = append(env, certEnv...) - } - if info.IdentifiersParseError != nil { - env = append(env, "IDENTIFIERS_PARSE_ERROR="+info.IdentifiersParseError.Error()) - } else if info.Identifiers != nil { - env = append(env, "DNS_NAMES="+info.Identifiers.dnsNamesString(",")) - env = append(env, "IP_ADDRESSES="+info.Identifiers.ipAddrsString(",")) - } - - return env -} - -func writeField(out io.Writer, name string, value interface{}, err error) { - if err == nil { - fmt.Fprintf(out, "\t%13s = %s\n", name, value) - } else { - fmt.Fprintf(out, "\t%13s = *** UNKNOWN (%s) ***\n", name, err) - } -} - -func (info *EntryInfo) Write(out io.Writer) { - fingerprint := info.Fingerprint() - fmt.Fprintf(out, "%s:\n", fingerprint) - if info.IdentifiersParseError != nil { - writeField(out, "Identifiers", nil, info.IdentifiersParseError) - } else if info.Identifiers != nil { - for _, dnsName := range info.Identifiers.DNSNames { - writeField(out, "DNS Name", dnsName, nil) - } - for _, ipaddr := range info.Identifiers.IPAddrs { - writeField(out, "IP Address", ipaddr, nil) - } - } - if info.ParseError != nil { - writeField(out, "Parse Error", "*** "+info.ParseError.Error()+" ***", nil) - } else if info.CertInfo != nil { - writeField(out, "Pubkey", info.CertInfo.PubkeyHash(), nil) - writeField(out, "Issuer", info.CertInfo.Issuer, info.CertInfo.IssuerParseError) - writeField(out, "Not Before", info.CertInfo.NotBefore(), info.CertInfo.ValidityParseError) - writeField(out, "Not After", info.CertInfo.NotAfter(), info.CertInfo.ValidityParseError) - if info.Bygone { - writeField(out, "BygoneSSL", "True", info.CertInfo.ValidityParseError) - } - } - writeField(out, "Log Entry", fmt.Sprintf("%d @ %s (%s)", info.Entry.Index, info.LogUri, info.typeFriendlyString()), nil) - writeField(out, "crt.sh", "https://crt.sh/?sha256="+fingerprint, nil) - if info.Filename != "" { - writeField(out, "Filename", info.Filename, nil) - } -} - -func (info *EntryInfo) InvokeHookScript(command string) error { - cmd := exec.Command(command) - cmd.Env = os.Environ() - infoEnv := info.Environ() - cmd.Env = append(cmd.Env, infoEnv...) - stderrBuffer := bytes.Buffer{} - cmd.Stderr = &stderrBuffer - if err := cmd.Run(); err != nil { - if _, isExitError := err.(*exec.ExitError); isExitError { - return fmt.Errorf("Script failed: %s: %s", command, strings.TrimSpace(stderrBuffer.String())) - } else { - return fmt.Errorf("Failed to execute script: %s: %s", command, err) - } - } - return nil -} - func MatchesWildcard(dnsName string, pattern string) bool { for len(pattern) > 0 { if pattern[0] == '*' { diff --git a/scanner.go b/scanner.go deleted file mode 100644 index 0f06939..0000000 --- a/scanner.go +++ /dev/null @@ -1,299 +0,0 @@ -// Copyright (C) 2016 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. -// -// This file contains code from https://github.com/google/certificate-transparency/tree/master/go -// See ct/AUTHORS and ct/LICENSE for copyright and license information. - -package certspotter - -import ( - // "container/list" - "bytes" - "context" - "crypto" - "errors" - "fmt" - "log" - "strings" - "sync" - "sync/atomic" - "time" - - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/ct/client" -) - -type ProcessCallback func(*Scanner, *ct.LogEntry) - -// ScannerOptions holds configuration options for the Scanner -type ScannerOptions struct { - // Number of entries to request in one batch from the Log - BatchSize int - - // Number of concurrent proecssors to run - NumWorkers int - - // Don't print any status messages to stdout - Quiet bool -} - -// Creates a new ScannerOptions struct with sensible defaults -func DefaultScannerOptions() *ScannerOptions { - return &ScannerOptions{ - BatchSize: 1000, - NumWorkers: 1, - Quiet: false, - } -} - -// Scanner is a tool to scan all the entries in a CT Log. -type Scanner struct { - // Base URI of CT log - LogUri string - - // Public key of the log - publicKey crypto.PublicKey - LogId ct.SHA256Hash - - // Client used to talk to the CT log instance - logClient *client.LogClient - - // Configuration options for this Scanner instance - opts ScannerOptions -} - -// fetchRange represents a range of certs to fetch from a CT log -type fetchRange struct { - start int64 - end int64 -} - -// Worker function to process certs. -// Accepts ct.LogEntries over the |entries| channel, and invokes processCert on them. -// Returns true over the |done| channel when the |entries| channel is closed. -func (s *Scanner) processerJob(id int, certsProcessed *int64, entries <-chan ct.LogEntry, processCert ProcessCallback, wg *sync.WaitGroup) { - for entry := range entries { - atomic.AddInt64(certsProcessed, 1) - processCert(s, &entry) - } - wg.Done() -} - -func (s *Scanner) fetch(r fetchRange, entries chan<- ct.LogEntry, tree *CollapsedMerkleTree) error { - for r.start <= r.end { - s.Log(fmt.Sprintf("Fetching entries %d to %d", r.start, r.end)) - logEntries, err := s.logClient.GetEntries(context.Background(), r.start, r.end) - if err != nil { - return err - } - for _, logEntry := range logEntries { - if tree != nil { - tree.Add(hashLeaf(logEntry.LeafBytes)) - } - logEntry.Index = r.start - entries <- logEntry - r.start++ - } - } - return nil -} - -// Worker function for fetcher jobs. -// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is -// successful sends the individual LeafInputs out into the -// |entries| channel for the processors to chew on. -// Will retry failed attempts to retrieve ranges indefinitely. -// Sends true over the |done| channel when the |ranges| channel is closed. -/* disabled becuase error handling is broken -func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- ct.LogEntry, wg *sync.WaitGroup) { - for r := range ranges { - s.fetch(r, entries, nil) - } - wg.Done() -} -*/ - -// Returns the smaller of |a| and |b| -func min(a int64, b int64) int64 { - if a < b { - return a - } else { - return b - } -} - -// Returns the larger of |a| and |b| -func max(a int64, b int64) int64 { - if a > b { - return a - } else { - return b - } -} - -// Pretty prints the passed in number of |seconds| into a more human readable -// string. -func humanTime(seconds int) string { - nanos := time.Duration(seconds) * time.Second - hours := int(nanos / (time.Hour)) - nanos %= time.Hour - minutes := int(nanos / time.Minute) - nanos %= time.Minute - seconds = int(nanos / time.Second) - s := "" - if hours > 0 { - s += fmt.Sprintf("%d hours ", hours) - } - if minutes > 0 { - s += fmt.Sprintf("%d minutes ", minutes) - } - if seconds > 0 { - s += fmt.Sprintf("%d seconds ", seconds) - } - return s -} - -func (s Scanner) Log(msg string) { - if !s.opts.Quiet { - log.Print(s.LogUri, ": ", msg) - } -} - -func (s *Scanner) GetSTH() (*ct.SignedTreeHead, error) { - latestSth, err := s.logClient.GetSTH(context.Background()) - if err != nil { - return nil, err - } - if s.publicKey != nil { - verifier, err := ct.NewSignatureVerifier(s.publicKey) - if err != nil { - return nil, err - } - if err := verifier.VerifySTHSignature(*latestSth); err != nil { - return nil, errors.New("STH signature is invalid: " + err.Error()) - } - } - latestSth.LogID = s.LogId - return latestSth, nil -} - -func (s *Scanner) CheckConsistency(first *ct.SignedTreeHead, second *ct.SignedTreeHead) (bool, error) { - if first.TreeSize == 0 || second.TreeSize == 0 { - // RFC 6962 doesn't define how to generate a consistency proof in this case, - // and it doesn't matter anyways since the tree is empty. The DigiCert logs - // return a 400 error if we ask for such a proof. - return true, nil - } else if first.TreeSize < second.TreeSize { - proof, err := s.logClient.GetConsistencyProof(context.Background(), int64(first.TreeSize), int64(second.TreeSize)) - if err != nil { - return false, err - } - return VerifyConsistencyProof(proof, first, second), nil - } else if first.TreeSize > second.TreeSize { - proof, err := s.logClient.GetConsistencyProof(context.Background(), int64(second.TreeSize), int64(first.TreeSize)) - if err != nil { - return false, err - } - return VerifyConsistencyProof(proof, second, first), nil - } else { - // There is no need to ask the server for a consistency proof if the trees - // are the same size, and the DigiCert log returns a 400 error if we try. - return bytes.Equal(first.SHA256RootHash[:], second.SHA256RootHash[:]), nil - } -} - -func (s *Scanner) MakeCollapsedMerkleTree(sth *ct.SignedTreeHead) (*CollapsedMerkleTree, error) { - if sth.TreeSize == 0 { - return &CollapsedMerkleTree{}, nil - } - - entries, err := s.logClient.GetEntries(context.Background(), int64(sth.TreeSize-1), int64(sth.TreeSize-1)) - if err != nil { - return nil, err - } - if len(entries) == 0 { - return nil, fmt.Errorf("Log did not return entry %d", sth.TreeSize-1) - } - leafHash := hashLeaf(entries[0].LeafBytes) - - var tree *CollapsedMerkleTree - if sth.TreeSize > 1 { - auditPath, _, err := s.logClient.GetAuditProof(context.Background(), leafHash, sth.TreeSize) - if err != nil { - return nil, err - } - reverseHashes(auditPath) - tree, err = NewCollapsedMerkleTree(auditPath, sth.TreeSize-1) - if err != nil { - return nil, fmt.Errorf("Error returned bad audit proof for %x to %d", leafHash, sth.TreeSize) - } - } else { - tree = EmptyCollapsedMerkleTree() - } - - tree.Add(leafHash) - if !bytes.Equal(tree.CalculateRoot(), sth.SHA256RootHash[:]) { - return nil, fmt.Errorf("Calculated root hash does not match signed tree head at size %d", sth.TreeSize) - } - - return tree, nil -} - -func (s *Scanner) Scan(startIndex int64, endIndex int64, processCert ProcessCallback, tree *CollapsedMerkleTree) error { - s.Log("Starting scan...") - - certsProcessed := new(int64) - startTime := time.Now() - /* TODO: only launch ticker goroutine if in verbose mode; kill the goroutine when the scanner finishes - ticker := time.NewTicker(time.Second) - go func() { - for range ticker.C { - throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds() - remainingCerts := int64(endIndex) - int64(startIndex) - s.certsProcessed - remainingSeconds := int(float64(remainingCerts) / throughput) - remainingString := humanTime(remainingSeconds) - s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s", s.certsProcessed, - startIndex+int64(s.certsProcessed), throughput, remainingString)) - } - }() - */ - - // Start processor workers - jobs := make(chan ct.LogEntry, 100) - var processorWG sync.WaitGroup - for w := 0; w < s.opts.NumWorkers; w++ { - processorWG.Add(1) - go s.processerJob(w, certsProcessed, jobs, processCert, &processorWG) - } - - for start := startIndex; start < int64(endIndex); { - end := min(start+int64(s.opts.BatchSize), int64(endIndex)) - 1 - if err := s.fetch(fetchRange{start, end}, jobs, tree); err != nil { - return err - } - start = end + 1 - } - close(jobs) - processorWG.Wait() - s.Log(fmt.Sprintf("Completed %d certs in %s", *certsProcessed, humanTime(int(time.Since(startTime).Seconds())))) - - return nil -} - -// Creates a new Scanner instance using |client| to talk to the log, and taking -// configuration options from |opts|. -func NewScanner(logUri string, logId ct.SHA256Hash, publicKey crypto.PublicKey, opts *ScannerOptions) *Scanner { - var scanner Scanner - scanner.LogUri = logUri - scanner.LogId = logId - scanner.publicKey = publicKey - scanner.logClient = client.New(strings.TrimRight(logUri, "/")) - scanner.opts = *opts - return &scanner -} diff --git a/version.go b/version.go deleted file mode 100644 index d2bc678..0000000 --- a/version.go +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (C) 2022 Opsmate, Inc. -// -// This Source Code Form is subject to the terms of the Mozilla -// Public License, v. 2.0. If a copy of the MPL was not distributed -// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. -// -// This software is distributed WITHOUT A WARRANTY OF ANY KIND. -// See the Mozilla Public License for details. - -package certspotter - -const Version = "0.14.0"