Tandem
/
certspotter
mirror of https://github.com/SSLMate/certspotter.git
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

New and simplified multi-log operation

This commit is contained in:
Andrew Ayer 2016-02-04 20:16:25 -08:00
parent a418a3686d
commit 3f596730a0
5 changed files with 148 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

160
cmd/common.go
View File

@ -5,7 +5,11 @@ import (
"fmt"
"log"
"os"
"os/user"
"bufio"
"sync"
"strings"
"path/filepath"
"src.agwa.name/ctwatch"
"github.com/google/certificate-transparency/go"
@ -16,14 +20,53 @@ var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request a
var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers")
var parallelFetch = flag.Int("parallel_fetch", 2, "Number of concurrent GetEntries fetches")
var script = flag.String("script", "", "Script to execute when a matching certificate is found")
var repo = flag.String("repo", "", "Directory of scanned certificates")
var logsFilename = flag.String("logs", "", "File containing log URLs")
var noSave = flag.Bool("no_save", false, "Do not save a copy of matching certificates")
var verbose = flag.Bool("verbose", false, "Be verbose")
var stateDir string
var printMutex sync.Mutex
func logCallback (entry *ct.LogEntry) {
if *repo != "" {
alreadyPresent, err := ctwatch.WriteCertRepository(*repo, entry)
var defaultLogs = []string{
"https://log.certly.io",
"https://ct1.digicert-ct.com/log",
"https://ct.googleapis.com/aviator",
"https://ct.googleapis.com/pilot",
"https://ct.googleapis.com/rocketeer",
"https://ct.izenpe.com",
"https://ct.ws.symantec.com",
"https://vega.ws.symantec.com",
"https://ctlog.api.venafi.com",
"https://ct.wosign.com",
}
func isRoot () bool {
return os.Geteuid() == 0
}
func homedir () string {
home := os.Getenv("HOME")
if home != "" {
return home
}
user, err := user.Current()
if err == nil {
return user.HomeDir
}
panic("Unable to determine home directory")
}
func DefaultStateDir (programName string) string {
if isRoot() {
return filepath.Join("/var/lib", programName)
} else {
return filepath.Join(homedir(), "." + programName)
}
}
func logCallback (scanner *ctwatch.Scanner, entry *ct.LogEntry) {
if !*noSave {
alreadyPresent, err := ctwatch.WriteCertRepository(filepath.Join(stateDir, "certs"), entry)
if err != nil {
log.Print(err)
}
@ -33,51 +76,96 @@ func logCallback (entry *ct.LogEntry) {
}
if *script != "" {
if err := ctwatch.InvokeHookScript(*script, entry); err != nil {
if err := ctwatch.InvokeHookScript(*script, scanner.LogUri, entry); err != nil {
log.Print(err)
}
} else {
printMutex.Lock()
ctwatch.DumpLogEntry(os.Stdout, entry)
ctwatch.DumpLogEntry(os.Stdout, scanner.LogUri, entry)
fmt.Fprintf(os.Stdout, "\n")
printMutex.Unlock()
}
}
func Main(logUri string, stateFile string, matcher ctwatch.Matcher) {
startIndex, err := ctwatch.ReadStateFile(stateFile)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: Error reading state file: %s: %s\n", os.Args[0], stateFile, err)
func defangLogUri (logUri string) string {
return strings.Replace(strings.Replace(logUri, "://", "_", 1), "/", "_", -1)
}
func Main (argStateDir string, matcher ctwatch.Matcher) {
stateDir = argStateDir
var logs []string
if *logsFilename != "" {
logFile, err := os.Open(*logsFilename)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: Error opening logs file for reading: %s: %s\n", os.Args[0], *logsFilename, err)
os.Exit(3)
}
defer logFile.Close()
scanner := bufio.NewScanner(logFile)
for scanner.Scan() {
logs = append(logs, scanner.Text())
}
if err := scanner.Err(); err != nil {
fmt.Fprintf(os.Stderr, "%s: Error reading logs file: %s: %s\n", os.Args[0], *logsFilename, err)
os.Exit(3)
}
} else {
logs = defaultLogs
}
if err := os.Mkdir(stateDir, 0777); err != nil && !os.IsExist(err) {
fmt.Fprintf(os.Stderr, "%s: Error creating state directory: %s: %s\n", os.Args[0], stateDir, err)
os.Exit(3)
}
os.Setenv("LOG_URI", logUri)
logClient := client.New(logUri)
opts := ctwatch.ScannerOptions{
Matcher: matcher,
BatchSize: *batchSize,
NumWorkers: *numWorkers,
ParallelFetch: *parallelFetch,
Quiet: !*verbose,
}
scanner := ctwatch.NewScanner(logClient, opts)
endIndex, err := scanner.TreeSize()
if err != nil {
fmt.Fprintf(os.Stderr, "%s: Error contacting log: %s: %s\n", os.Args[0], logUri, err)
os.Exit(1)
}
if startIndex != -1 {
if err := scanner.Scan(startIndex, endIndex, logCallback); err != nil {
fmt.Fprintf(os.Stderr, "%s: Error scanning log: %s: %s\n", os.Args[0], logUri, err)
os.Exit(1)
for _, subdir := range []string{"certs", "logs"} {
path := filepath.Join(stateDir, subdir)
if err := os.Mkdir(path, 0777); err != nil && !os.IsExist(err) {
fmt.Fprintf(os.Stderr, "%s: Error creating state directory: %s: %s\n", os.Args[0], path, err)
os.Exit(3)
}
}
if err := ctwatch.WriteStateFile(stateFile, endIndex); err != nil {
fmt.Fprintf(os.Stderr, "%s: Error writing state file: %s: %s\n", os.Args[0], stateFile, err)
os.Exit(3)
exitCode := 0
for _, logUri := range logs {
stateFilename := filepath.Join(stateDir, "logs", defangLogUri(logUri))
startIndex, err := ctwatch.ReadStateFile(stateFilename)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: Error reading state file: %s: %s\n", os.Args[0], stateFilename, err)
os.Exit(3)
}
logClient := client.New(logUri)
opts := ctwatch.ScannerOptions{
Matcher: matcher,
BatchSize: *batchSize,
NumWorkers: *numWorkers,
ParallelFetch: *parallelFetch,
Quiet: !*verbose,
}
scanner := ctwatch.NewScanner(logUri, logClient, opts)
endIndex, err := scanner.TreeSize()
if err != nil {
fmt.Fprintf(os.Stderr, "%s: Error contacting log: %s: %s\n", os.Args[0], logUri, err)
exitCode = 1
continue
}
if startIndex != -1 {
if err := scanner.Scan(startIndex, endIndex, logCallback); err != nil {
fmt.Fprintf(os.Stderr, "%s: Error scanning log: %s: %s\n", os.Args[0], logUri, err)
exitCode = 1
continue
}
}
if err := ctwatch.WriteStateFile(stateFilename, endIndex); err != nil {
fmt.Fprintf(os.Stderr, "%s: Error writing state file: %s: %s\n", os.Args[0], stateFilename, err)
os.Exit(3)
}
}
os.Exit(exitCode)
}

19
cmd/ctwatch/main.go
View File

@ -10,28 +10,23 @@ import (
"src.agwa.name/ctwatch/cmd"
)
var stateDir = flag.String("state_dir", cmd.DefaultStateDir("ctwatch"), "Directory for storing state")
func main() {
flag.Parse()
if flag.NArg() < 2 {
fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file [domain ...]\n", os.Args[0])
os.Exit(2)
}
logUri := flag.Arg(0)
stateFile := flag.Arg(1)
var domains []string
if flag.NArg() == 3 && flag.Arg(2) == "-" {
if flag.NArg() == 1 && flag.Arg(0) == "-" {
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
domains = append(domains, scanner.Text())
}
if err := scanner.Err(); err != nil {
fmt.Fprintf(os.Stderr, "Error reading standard input: %s\n", err)
os.Exit(1)
fmt.Fprintf(os.Stderr, "%s: Error reading standard input: %s\n", os.Args[0], err)
os.Exit(3)
}
} else {
domains = flag.Args()[2:]
domains = flag.Args()
}
var matcher ctwatch.Matcher
@ -41,5 +36,5 @@ func main() {
matcher = ctwatch.NewDomainMatcher(domains)
}
cmd.Main(logUri, stateFile, matcher)
cmd.Main(*stateDir, matcher)
}

13
cmd/sha1watch/main.go
View File

@ -2,8 +2,6 @@ package main
import (
"flag"
"fmt"
"os"
"time"
"github.com/google/certificate-transparency/go"
@ -12,6 +10,8 @@ import (
"src.agwa.name/ctwatch/cmd"
)
var stateDir = flag.String("state_dir", cmd.DefaultStateDir("sha1watch"), "Directory for storing state")
type sha1Matcher struct { }
func (m sha1Matcher) CertificateMatches(c *x509.Certificate) bool {
@ -29,13 +29,6 @@ func (m sha1Matcher) PrecertificateMatches(pc *ct.Precertificate) bool {
func main() {
flag.Parse()
if flag.NArg() != 2 {
fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file\n", os.Args[0])
os.Exit(2)
}
logUri := flag.Arg(0)
stateFile := flag.Arg(1)
cmd.Main(logUri, stateFile, &sha1Matcher{})
cmd.Main(*stateDir, &sha1Matcher{})
}

7
helpers.go
View File

@ -185,10 +185,10 @@ func (info *certInfo) TypeFriendlyString () string {
}
}
func DumpLogEntry (out io.Writer, entry *ct.LogEntry) {
func DumpLogEntry (out io.Writer, logUri string, entry *ct.LogEntry) {
info := makeCertInfo(entry)
fmt.Fprintf(out, "%d:\n", entry.Index)
fmt.Fprintf(out, "%d @ %s:\n", entry.Index, logUri)
fmt.Fprintf(out, "\t Type = %s\n", info.TypeFriendlyString())
fmt.Fprintf(out, "\t DNS Names = %v\n", info.DnsNames)
fmt.Fprintf(out, "\t Pubkey = %s\n", info.PubkeyHash)
@ -201,11 +201,12 @@ func DumpLogEntry (out io.Writer, entry *ct.LogEntry) {
fmt.Fprintf(out, "\t Not After = %s\n", info.NotAfter)
}
func InvokeHookScript (command string, entry *ct.LogEntry) error {
func InvokeHookScript (command string, logUri string, entry *ct.LogEntry) error {
info := makeCertInfo(entry)
cmd := exec.Command(command)
cmd.Env = append(os.Environ(),
"LOG_URI=" + logUri,
"LOG_INDEX=" + strconv.FormatInt(entry.Index, 10),
"CERT_TYPE=" + info.TypeString(),
"SUBJECT_DN=" + info.SubjectDn,

16
scanner.go
View File

@ -114,6 +114,9 @@ func DefaultScannerOptions() *ScannerOptions {
// Scanner is a tool to scan all the entries in a CT Log.
type Scanner struct {
// Base URI of CT log
LogUri string
// Client used to talk to the CT log instance
logClient *client.LogClient
@ -170,7 +173,7 @@ func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, in
}
// Processes the given |entry| in the specified log.
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry)) {
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*Scanner, *ct.LogEntry)) {
atomic.AddInt64(&s.certsProcessed, 1)
switch entry.Leaf.TimestampedEntry.EntryType {
case ct.X509LogEntryType:
@ -181,7 +184,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry))
}
if s.opts.Matcher.CertificateMatches(cert) {
entry.X509Cert = cert
foundCert(&entry)
foundCert(s, &entry)
}
case ct.PrecertLogEntryType:
c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
@ -196,7 +199,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry))
}
if s.opts.Matcher.PrecertificateMatches(precert) {
entry.Precert = precert
foundCert(&entry)
foundCert(s, &entry)
}
}
}
@ -204,7 +207,7 @@ func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry))
// Worker function to match certs.
// Accepts MatcherJobs over the |entries| channel, and processes them.
// Returns true over the |done| channel when the |entries| channel is closed.
func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry), wg *sync.WaitGroup) {
func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*Scanner, *ct.LogEntry), wg *sync.WaitGroup) {
for e := range entries {
s.processEntry(e.entry, foundCert)
}
@ -304,7 +307,7 @@ func (s *Scanner) TreeSize() (int64, error) {
return int64(latestSth.TreeSize), nil
}
func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*ct.LogEntry)) error {
func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*Scanner, *ct.LogEntry)) error {
s.Log("Starting up...\n")
s.certsProcessed = 0
@ -358,8 +361,9 @@ func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*ct.LogE
// Creates a new Scanner instance using |client| to talk to the log, and taking
// configuration options from |opts|.
func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner {
func NewScanner(logUri string, client *client.LogClient, opts ScannerOptions) *Scanner {
var scanner Scanner
scanner.LogUri = logUri
scanner.logClient = client
// Set a default match-everything regex if none was provided:
if opts.Matcher == nil {