New and simplified multi-log operation

cmd/common.go

@@ -5,7 +5,11 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"os/user"
+	"bufio"
 	"sync"
+	"strings"
+	"path/filepath"
 
 	"src.agwa.name/ctwatch"
 	"github.com/google/certificate-transparency/go"
@@ -16,14 +20,53 @@ var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request a
 var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers")
 var parallelFetch = flag.Int("parallel_fetch", 2, "Number of concurrent GetEntries fetches")
 var script = flag.String("script", "", "Script to execute when a matching certificate is found")
-var repo = flag.String("repo", "", "Directory of scanned certificates")
+var logsFilename = flag.String("logs", "", "File containing log URLs")
+var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates")
 var verbose = flag.Bool("verbose", false, "Be verbose")
+var stateDir string
 
 var printMutex sync.Mutex
 
-func logCallback (entry *ct.LogEntry) {
+var defaultLogs = []string{
-	if *repo != "" {
+	"https://log.certly.io",
-		alreadyPresent, err := ctwatch.WriteCertRepository(*repo, entry)
+	"https://ct1.digicert-ct.com/log",
+	"https://ct.googleapis.com/aviator",
+	"https://ct.googleapis.com/pilot",
+	"https://ct.googleapis.com/rocketeer",
+	"https://ct.izenpe.com",
+	"https://ct.ws.symantec.com",
+	"https://vega.ws.symantec.com",
+	"https://ctlog.api.venafi.com",
+	"https://ct.wosign.com",
+}
+
+func isRoot () bool {
+	return os.Geteuid() == 0
+}
+
+func homedir () string {
+	home := os.Getenv("HOME")
+	if home != "" {
+		return home
+	}
+	user, err := user.Current()
+	if err == nil {
+		return user.HomeDir
+	}
+	panic("Unable to determine home directory")
+}
+
+func DefaultStateDir (programName string) string {
+	if isRoot() {
+		return filepath.Join("/var/lib", programName)
+	} else {
+		return filepath.Join(homedir(), "." + programName)
+	}
+}
+
+func logCallback (scanner *ctwatch.Scanner, entry *ct.LogEntry) {
+	if !*noSave {
+		alreadyPresent, err := ctwatch.WriteCertRepository(filepath.Join(stateDir, "certs"), entry)
 		if err != nil {
 			log.Print(err)
 		}
@@ -33,25 +76,65 @@ func logCallback (entry *ct.LogEntry) {
 	}
 
 	if *script != "" {
-		if err := ctwatch.InvokeHookScript(*script, entry); err != nil {
+		if err := ctwatch.InvokeHookScript(*script, scanner.LogUri, entry); err != nil {
 			log.Print(err)
 		}
 	} else {
 		printMutex.Lock()
-		ctwatch.DumpLogEntry(os.Stdout, entry)
+		ctwatch.DumpLogEntry(os.Stdout, scanner.LogUri, entry)
 		fmt.Fprintf(os.Stdout, "\n")
 		printMutex.Unlock()
 	}
 }
 
-func Main(logUri string, stateFile string, matcher ctwatch.Matcher) {
+func defangLogUri (logUri string) string {
-	startIndex, err := ctwatch.ReadStateFile(stateFile)
+	return strings.Replace(strings.Replace(logUri, "://", "_", 1), "/", "_", -1)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "%s: Error reading state file: %s: %s\n", os.Args[0], stateFile, err)
-		os.Exit(3)
 }
 
-	os.Setenv("LOG_URI", logUri)
+func Main (argStateDir string, matcher ctwatch.Matcher) {
+	stateDir = argStateDir
+
+	var logs []string
+	if *logsFilename != "" {
+		logFile, err := os.Open(*logsFilename)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: Error opening logs file for reading: %s: %s\n", os.Args[0], *logsFilename, err)
+			os.Exit(3)
+		}
+		defer logFile.Close()
+		scanner := bufio.NewScanner(logFile)
+		for scanner.Scan() {
+			logs = append(logs, scanner.Text())
+		}
+		if err := scanner.Err(); err != nil {
+			fmt.Fprintf(os.Stderr, "%s: Error reading logs file: %s: %s\n", os.Args[0], *logsFilename, err)
+			os.Exit(3)
+		}
+	} else {
+		logs = defaultLogs
+	}
+
+	if err := os.Mkdir(stateDir, 0777); err != nil && !os.IsExist(err) {
+		fmt.Fprintf(os.Stderr, "%s: Error creating state directory: %s: %s\n", os.Args[0], stateDir, err)
+		os.Exit(3)
+	}
+	for _, subdir := range []string{"certs", "logs"} {
+		path := filepath.Join(stateDir, subdir)
+		if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) {
+			fmt.Fprintf(os.Stderr, "%s: Error creating state directory: %s: %s\n", os.Args[0], path, err)
+			os.Exit(3)
+		}
+	}
+
+	exitCode := 0
+
+	for _, logUri := range logs {
+		stateFilename := filepath.Join(stateDir, "logs", defangLogUri(logUri))
+		startIndex, err := ctwatch.ReadStateFile(stateFilename)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "%s: Error reading state file: %s: %s\n", os.Args[0], stateFilename, err)
+			os.Exit(3)
+		}
 
 		logClient := client.New(logUri)
 		opts := ctwatch.ScannerOptions{
@@ -61,23 +144,28 @@ func Main(logUri string, stateFile string, matcher ctwatch.Matcher) {
 			ParallelFetch: *parallelFetch,
 			Quiet:         !*verbose,
 		}
-	scanner := ctwatch.NewScanner(logClient, opts)
+		scanner := ctwatch.NewScanner(logUri, logClient, opts)
 
 		endIndex, err := scanner.TreeSize()
 		if err != nil {
 			fmt.Fprintf(os.Stderr, "%s: Error contacting log: %s: %s\n", os.Args[0], logUri, err)
-		os.Exit(1)
+			exitCode = 1
+			continue
 		}
 
 		if startIndex != -1 {
 			if err := scanner.Scan(startIndex, endIndex, logCallback); err != nil {
 				fmt.Fprintf(os.Stderr, "%s: Error scanning log: %s: %s\n", os.Args[0], logUri, err)
-			os.Exit(1)
+				exitCode = 1
+				continue
 			}
 		}
 
-	if err := ctwatch.WriteStateFile(stateFile, endIndex); err != nil {
+		if err := ctwatch.WriteStateFile(stateFilename, endIndex); err != nil {
-		fmt.Fprintf(os.Stderr, "%s: Error writing state file: %s: %s\n", os.Args[0], stateFile, err)
+			fmt.Fprintf(os.Stderr, "%s: Error writing state file: %s: %s\n", os.Args[0], stateFilename, err)
 			os.Exit(3)
 		}
 	}
+
+	os.Exit(exitCode)
+}

cmd/ctwatch/main.go

@@ -10,28 +10,23 @@ import (
 	"src.agwa.name/ctwatch/cmd"
 )
 
+var stateDir = flag.String("state_dir", cmd.DefaultStateDir("ctwatch"), "Directory for storing state")
+
 func main() {
 	flag.Parse()
-	if flag.NArg() < 2 {
-		fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file [domain ...]\n", os.Args[0])
-		os.Exit(2)
-	}
-
-	logUri := flag.Arg(0)
-	stateFile := flag.Arg(1)
 
 	var domains []string
-	if flag.NArg() == 3 && flag.Arg(2) == "-" {
+	if flag.NArg() == 1 && flag.Arg(0) == "-" {
 		scanner := bufio.NewScanner(os.Stdin)
 		for scanner.Scan() {
 			domains = append(domains, scanner.Text())
 		}
 		if err := scanner.Err(); err != nil {
-			fmt.Fprintf(os.Stderr, "Error reading standard input: %s\n", err)
+			fmt.Fprintf(os.Stderr, "%s: Error reading standard input: %s\n", os.Args[0], err)
-			os.Exit(1)
+			os.Exit(3)
 		}
 	} else {
-		domains = flag.Args()[2:]
+		domains = flag.Args()
 	}
 
 	var matcher ctwatch.Matcher
@@ -41,5 +36,5 @@ func main() {
 		matcher = ctwatch.NewDomainMatcher(domains)
 	}
 
-	cmd.Main(logUri, stateFile, matcher)
+	cmd.Main(*stateDir, matcher)
 }

cmd/sha1watch/main.go

@@ -2,8 +2,6 @@ package main
 
 import (
 	"flag"
-	"fmt"
-	"os"
 	"time"
 
 	"github.com/google/certificate-transparency/go"
@@ -12,6 +10,8 @@ import (
 	"src.agwa.name/ctwatch/cmd"
 )
 
+var stateDir = flag.String("state_dir", cmd.DefaultStateDir("sha1watch"), "Directory for storing state")
+
 type sha1Matcher struct { }
 
 func (m sha1Matcher) CertificateMatches(c *x509.Certificate) bool {
@@ -29,13 +29,6 @@ func (m sha1Matcher) PrecertificateMatches(pc *ct.Precertificate) bool {
 
 func main() {
 	flag.Parse()
-	if flag.NArg() != 2 {
-		fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file\n", os.Args[0])
-		os.Exit(2)
-	}
 
-	logUri := flag.Arg(0)
+	cmd.Main(*stateDir, &sha1Matcher{})
-	stateFile := flag.Arg(1)
-
-	cmd.Main(logUri, stateFile, &sha1Matcher{})
 }

helpers.go

@@ -185,10 +185,10 @@ func (info *certInfo) TypeFriendlyString () string {
 	}
 }
 
-func DumpLogEntry (out io.Writer, entry *ct.LogEntry) {
+func DumpLogEntry (out io.Writer, logUri string, entry *ct.LogEntry) {
 	info := makeCertInfo(entry)
 
-	fmt.Fprintf(out, "%d:\n", entry.Index)
+	fmt.Fprintf(out, "%d @ %s:\n", entry.Index, logUri)
 	fmt.Fprintf(out, "\t         Type = %s\n", info.TypeFriendlyString())
 	fmt.Fprintf(out, "\t    DNS Names = %v\n", info.DnsNames)
 	fmt.Fprintf(out, "\t       Pubkey = %s\n", info.PubkeyHash)
@@ -201,11 +201,12 @@ func DumpLogEntry (out io.Writer, entry *ct.LogEntry) {
 	fmt.Fprintf(out, "\t    Not After = %s\n", info.NotAfter)
 }
 
-func InvokeHookScript (command string, entry *ct.LogEntry) error {
+func InvokeHookScript (command string, logUri string, entry *ct.LogEntry) error {
 	info := makeCertInfo(entry)
 
 	cmd := exec.Command(command)
 	cmd.Env = append(os.Environ(),
+				"LOG_URI=" + logUri,
 				"LOG_INDEX=" + strconv.FormatInt(entry.Index, 10),
 				"CERT_TYPE=" + info.TypeString(),
 				"SUBJECT_DN=" + info.SubjectDn,

scanner.go

@@ -114,6 +114,9 @@ func DefaultScannerOptions() *ScannerOptions {
 
 // Scanner is a tool to scan all the entries in a CT Log.
 type Scanner struct {
+	// Base URI of CT log
+	LogUri				string
+
 	// Client used to talk to the CT log instance
 	logClient			*client.LogClient
 
@@ -170,7 +173,7 @@ func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, in
 }
 
 // Processes the given |entry| in the specified log.
-func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry)) {
+func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*Scanner, *ct.LogEntry)) {
 	atomic.AddInt64(&s.certsProcessed, 1)
 	switch entry.Leaf.TimestampedEntry.EntryType {
 	case ct.X509LogEntryType:
@@ -181,7 +184,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry))
 		}
 		if s.opts.Matcher.CertificateMatches(cert) {
 			entry.X509Cert = cert
-			foundCert(&entry)
+			foundCert(s, &entry)
 		}
 	case ct.PrecertLogEntryType:
 		c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
@@ -196,7 +199,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry))
 		}
 		if s.opts.Matcher.PrecertificateMatches(precert) {
 			entry.Precert = precert
-			foundCert(&entry)
+			foundCert(s, &entry)
 		}
 	}
 }
@@ -204,7 +207,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry))
 // Worker function to match certs.
 // Accepts MatcherJobs over the |entries| channel, and processes them.
 // Returns true over the |done| channel when the |entries| channel is closed.
-func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry), wg *sync.WaitGroup) {
+func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*Scanner, *ct.LogEntry), wg *sync.WaitGroup) {
 	for e := range entries {
 		s.processEntry(e.entry, foundCert)
 	}
@@ -304,7 +307,7 @@ func (s *Scanner) TreeSize() (int64, error) {
 	return int64(latestSth.TreeSize), nil
 }
 
-func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*ct.LogEntry)) error {
+func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*Scanner, *ct.LogEntry)) error {
 	s.Log("Starting up...\n")
 
 	s.certsProcessed = 0
@@ -358,8 +361,9 @@ func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*ct.LogE
 
 // Creates a new Scanner instance using |client| to talk to the log, and taking
 // configuration options from |opts|.
-func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner {
+func NewScanner(logUri string, client *client.LogClient, opts ScannerOptions) *Scanner {
 	var scanner Scanner
+	scanner.LogUri = logUri
 	scanner.logClient = client
 	// Set a default match-everything regex if none was provided:
 	if opts.Matcher == nil {