From 3f596730a02db7d881129addc40b818f708f7cb1 Mon Sep 17 00:00:00 2001 From: Andrew Ayer Date: Thu, 4 Feb 2016 20:16:25 -0800 Subject: [PATCH] New and simplified multi-log operation --- cmd/common.go | 160 ++++++++++++++++++++++++++++++++---------- cmd/ctwatch/main.go | 19 ++--- cmd/sha1watch/main.go | 13 +--- helpers.go | 7 +- scanner.go | 16 +++-- 5 files changed, 148 insertions(+), 67 deletions(-) diff --git a/cmd/common.go b/cmd/common.go index 0179221..5ac57b6 100644 --- a/cmd/common.go +++ b/cmd/common.go @@ -5,7 +5,11 @@ import ( "fmt" "log" "os" + "os/user" + "bufio" "sync" + "strings" + "path/filepath" "src.agwa.name/ctwatch" "github.com/google/certificate-transparency/go" @@ -16,14 +20,53 @@ var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request a var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers") var parallelFetch = flag.Int("parallel_fetch", 2, "Number of concurrent GetEntries fetches") var script = flag.String("script", "", "Script to execute when a matching certificate is found") -var repo = flag.String("repo", "", "Directory of scanned certificates") +var logsFilename = flag.String("logs", "", "File containing log URLs") +var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates") var verbose = flag.Bool("verbose", false, "Be verbose") +var stateDir string var printMutex sync.Mutex -func logCallback (entry *ct.LogEntry) { - if *repo != "" { - alreadyPresent, err := ctwatch.WriteCertRepository(*repo, entry) +var defaultLogs = []string{ + "https://log.certly.io", + "https://ct1.digicert-ct.com/log", + "https://ct.googleapis.com/aviator", + "https://ct.googleapis.com/pilot", + "https://ct.googleapis.com/rocketeer", + "https://ct.izenpe.com", + "https://ct.ws.symantec.com", + "https://vega.ws.symantec.com", + "https://ctlog.api.venafi.com", + "https://ct.wosign.com", +} + +func isRoot () bool { + return os.Geteuid() == 0 +} + +func homedir () string { + home := os.Getenv("HOME") + if home != "" { + return home + } + user, err := user.Current() + if err == nil { + return user.HomeDir + } + panic("Unable to determine home directory") +} + +func DefaultStateDir (programName string) string { + if isRoot() { + return filepath.Join("/var/lib", programName) + } else { + return filepath.Join(homedir(), "." + programName) + } +} + +func logCallback (scanner *ctwatch.Scanner, entry *ct.LogEntry) { + if !*noSave { + alreadyPresent, err := ctwatch.WriteCertRepository(filepath.Join(stateDir, "certs"), entry) if err != nil { log.Print(err) } @@ -33,51 +76,96 @@ func logCallback (entry *ct.LogEntry) { } if *script != "" { - if err := ctwatch.InvokeHookScript(*script, entry); err != nil { + if err := ctwatch.InvokeHookScript(*script, scanner.LogUri, entry); err != nil { log.Print(err) } } else { printMutex.Lock() - ctwatch.DumpLogEntry(os.Stdout, entry) + ctwatch.DumpLogEntry(os.Stdout, scanner.LogUri, entry) fmt.Fprintf(os.Stdout, "\n") printMutex.Unlock() } } -func Main(logUri string, stateFile string, matcher ctwatch.Matcher) { - startIndex, err := ctwatch.ReadStateFile(stateFile) - if err != nil { - fmt.Fprintf(os.Stderr, "%s: Error reading state file: %s: %s\n", os.Args[0], stateFile, err) +func defangLogUri (logUri string) string { + return strings.Replace(strings.Replace(logUri, "://", "_", 1), "/", "_", -1) +} + +func Main (argStateDir string, matcher ctwatch.Matcher) { + stateDir = argStateDir + + var logs []string + if *logsFilename != "" { + logFile, err := os.Open(*logsFilename) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: Error opening logs file for reading: %s: %s\n", os.Args[0], *logsFilename, err) + os.Exit(3) + } + defer logFile.Close() + scanner := bufio.NewScanner(logFile) + for scanner.Scan() { + logs = append(logs, scanner.Text()) + } + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "%s: Error reading logs file: %s: %s\n", os.Args[0], *logsFilename, err) + os.Exit(3) + } + } else { + logs = defaultLogs + } + + if err := os.Mkdir(stateDir, 0777); err != nil && !os.IsExist(err) { + fmt.Fprintf(os.Stderr, "%s: Error creating state directory: %s: %s\n", os.Args[0], stateDir, err) os.Exit(3) } - - os.Setenv("LOG_URI", logUri) - - logClient := client.New(logUri) - opts := ctwatch.ScannerOptions{ - Matcher: matcher, - BatchSize: *batchSize, - NumWorkers: *numWorkers, - ParallelFetch: *parallelFetch, - Quiet: !*verbose, - } - scanner := ctwatch.NewScanner(logClient, opts) - - endIndex, err := scanner.TreeSize() - if err != nil { - fmt.Fprintf(os.Stderr, "%s: Error contacting log: %s: %s\n", os.Args[0], logUri, err) - os.Exit(1) - } - - if startIndex != -1 { - if err := scanner.Scan(startIndex, endIndex, logCallback); err != nil { - fmt.Fprintf(os.Stderr, "%s: Error scanning log: %s: %s\n", os.Args[0], logUri, err) - os.Exit(1) + for _, subdir := range []string{"certs", "logs"} { + path := filepath.Join(stateDir, subdir) + if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) { + fmt.Fprintf(os.Stderr, "%s: Error creating state directory: %s: %s\n", os.Args[0], path, err) + os.Exit(3) } } - if err := ctwatch.WriteStateFile(stateFile, endIndex); err != nil { - fmt.Fprintf(os.Stderr, "%s: Error writing state file: %s: %s\n", os.Args[0], stateFile, err) - os.Exit(3) + exitCode := 0 + + for _, logUri := range logs { + stateFilename := filepath.Join(stateDir, "logs", defangLogUri(logUri)) + startIndex, err := ctwatch.ReadStateFile(stateFilename) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: Error reading state file: %s: %s\n", os.Args[0], stateFilename, err) + os.Exit(3) + } + + logClient := client.New(logUri) + opts := ctwatch.ScannerOptions{ + Matcher: matcher, + BatchSize: *batchSize, + NumWorkers: *numWorkers, + ParallelFetch: *parallelFetch, + Quiet: !*verbose, + } + scanner := ctwatch.NewScanner(logUri, logClient, opts) + + endIndex, err := scanner.TreeSize() + if err != nil { + fmt.Fprintf(os.Stderr, "%s: Error contacting log: %s: %s\n", os.Args[0], logUri, err) + exitCode = 1 + continue + } + + if startIndex != -1 { + if err := scanner.Scan(startIndex, endIndex, logCallback); err != nil { + fmt.Fprintf(os.Stderr, "%s: Error scanning log: %s: %s\n", os.Args[0], logUri, err) + exitCode = 1 + continue + } + } + + if err := ctwatch.WriteStateFile(stateFilename, endIndex); err != nil { + fmt.Fprintf(os.Stderr, "%s: Error writing state file: %s: %s\n", os.Args[0], stateFilename, err) + os.Exit(3) + } } + + os.Exit(exitCode) } diff --git a/cmd/ctwatch/main.go b/cmd/ctwatch/main.go index 7b7d82a..128f895 100644 --- a/cmd/ctwatch/main.go +++ b/cmd/ctwatch/main.go @@ -10,28 +10,23 @@ import ( "src.agwa.name/ctwatch/cmd" ) +var stateDir = flag.String("state_dir", cmd.DefaultStateDir("ctwatch"), "Directory for storing state") + func main() { flag.Parse() - if flag.NArg() < 2 { - fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file [domain ...]\n", os.Args[0]) - os.Exit(2) - } - - logUri := flag.Arg(0) - stateFile := flag.Arg(1) var domains []string - if flag.NArg() == 3 && flag.Arg(2) == "-" { + if flag.NArg() == 1 && flag.Arg(0) == "-" { scanner := bufio.NewScanner(os.Stdin) for scanner.Scan() { domains = append(domains, scanner.Text()) } if err := scanner.Err(); err != nil { - fmt.Fprintf(os.Stderr, "Error reading standard input: %s\n", err) - os.Exit(1) + fmt.Fprintf(os.Stderr, "%s: Error reading standard input: %s\n", os.Args[0], err) + os.Exit(3) } } else { - domains = flag.Args()[2:] + domains = flag.Args() } var matcher ctwatch.Matcher @@ -41,5 +36,5 @@ func main() { matcher = ctwatch.NewDomainMatcher(domains) } - cmd.Main(logUri, stateFile, matcher) + cmd.Main(*stateDir, matcher) } diff --git a/cmd/sha1watch/main.go b/cmd/sha1watch/main.go index d937b13..21163f6 100644 --- a/cmd/sha1watch/main.go +++ b/cmd/sha1watch/main.go @@ -2,8 +2,6 @@ package main import ( "flag" - "fmt" - "os" "time" "github.com/google/certificate-transparency/go" @@ -12,6 +10,8 @@ import ( "src.agwa.name/ctwatch/cmd" ) +var stateDir = flag.String("state_dir", cmd.DefaultStateDir("sha1watch"), "Directory for storing state") + type sha1Matcher struct { } func (m sha1Matcher) CertificateMatches(c *x509.Certificate) bool { @@ -29,13 +29,6 @@ func (m sha1Matcher) PrecertificateMatches(pc *ct.Precertificate) bool { func main() { flag.Parse() - if flag.NArg() != 2 { - fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file\n", os.Args[0]) - os.Exit(2) - } - logUri := flag.Arg(0) - stateFile := flag.Arg(1) - - cmd.Main(logUri, stateFile, &sha1Matcher{}) + cmd.Main(*stateDir, &sha1Matcher{}) } diff --git a/helpers.go b/helpers.go index a528841..26fa74d 100644 --- a/helpers.go +++ b/helpers.go @@ -185,10 +185,10 @@ func (info *certInfo) TypeFriendlyString () string { } } -func DumpLogEntry (out io.Writer, entry *ct.LogEntry) { +func DumpLogEntry (out io.Writer, logUri string, entry *ct.LogEntry) { info := makeCertInfo(entry) - fmt.Fprintf(out, "%d:\n", entry.Index) + fmt.Fprintf(out, "%d @ %s:\n", entry.Index, logUri) fmt.Fprintf(out, "\t Type = %s\n", info.TypeFriendlyString()) fmt.Fprintf(out, "\t DNS Names = %v\n", info.DnsNames) fmt.Fprintf(out, "\t Pubkey = %s\n", info.PubkeyHash) @@ -201,11 +201,12 @@ func DumpLogEntry (out io.Writer, entry *ct.LogEntry) { fmt.Fprintf(out, "\t Not After = %s\n", info.NotAfter) } -func InvokeHookScript (command string, entry *ct.LogEntry) error { +func InvokeHookScript (command string, logUri string, entry *ct.LogEntry) error { info := makeCertInfo(entry) cmd := exec.Command(command) cmd.Env = append(os.Environ(), + "LOG_URI=" + logUri, "LOG_INDEX=" + strconv.FormatInt(entry.Index, 10), "CERT_TYPE=" + info.TypeString(), "SUBJECT_DN=" + info.SubjectDn, diff --git a/scanner.go b/scanner.go index eb50e7a..3d78419 100644 --- a/scanner.go +++ b/scanner.go @@ -114,6 +114,9 @@ func DefaultScannerOptions() *ScannerOptions { // Scanner is a tool to scan all the entries in a CT Log. type Scanner struct { + // Base URI of CT log + LogUri string + // Client used to talk to the CT log instance logClient *client.LogClient @@ -170,7 +173,7 @@ func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, in } // Processes the given |entry| in the specified log. -func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry)) { +func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*Scanner, *ct.LogEntry)) { atomic.AddInt64(&s.certsProcessed, 1) switch entry.Leaf.TimestampedEntry.EntryType { case ct.X509LogEntryType: @@ -181,7 +184,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry)) } if s.opts.Matcher.CertificateMatches(cert) { entry.X509Cert = cert - foundCert(&entry) + foundCert(s, &entry) } case ct.PrecertLogEntryType: c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate) @@ -196,7 +199,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry)) } if s.opts.Matcher.PrecertificateMatches(precert) { entry.Precert = precert - foundCert(&entry) + foundCert(s, &entry) } } } @@ -204,7 +207,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry)) // Worker function to match certs. // Accepts MatcherJobs over the |entries| channel, and processes them. // Returns true over the |done| channel when the |entries| channel is closed. -func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry), wg *sync.WaitGroup) { +func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*Scanner, *ct.LogEntry), wg *sync.WaitGroup) { for e := range entries { s.processEntry(e.entry, foundCert) } @@ -304,7 +307,7 @@ func (s *Scanner) TreeSize() (int64, error) { return int64(latestSth.TreeSize), nil } -func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*ct.LogEntry)) error { +func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*Scanner, *ct.LogEntry)) error { s.Log("Starting up...\n") s.certsProcessed = 0 @@ -358,8 +361,9 @@ func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*ct.LogE // Creates a new Scanner instance using |client| to talk to the log, and taking // configuration options from |opts|. -func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner { +func NewScanner(logUri string, client *client.LogClient, opts ScannerOptions) *Scanner { var scanner Scanner + scanner.LogUri = logUri scanner.logClient = client // Set a default match-everything regex if none was provided: if opts.Matcher == nil {