Initial commit
This commit is contained in:
Andrew Ayer
commit
a418a3686d
|
|
@ -0,0 +1,3 @@
|
||||||
|
*/*
|
||||||
|
!.gitignore
|
||||||
|
!*.go
|
||||||
|
|
@ -0,0 +1,83 @@
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"src.agwa.name/ctwatch"
|
||||||
|
"github.com/google/certificate-transparency/go"
|
||||||
|
"github.com/google/certificate-transparency/go/client"
|
||||||
|
)
|
||||||
|
|
||||||
|
var batchSize = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries")
|
||||||
|
var numWorkers = flag.Int("num_workers", 2, "Number of concurrent matchers")
|
||||||
|
var parallelFetch = flag.Int("parallel_fetch", 2, "Number of concurrent GetEntries fetches")
|
||||||
|
var script = flag.String("script", "", "Script to execute when a matching certificate is found")
|
||||||
|
var repo = flag.String("repo", "", "Directory of scanned certificates")
|
||||||
|
var verbose = flag.Bool("verbose", false, "Be verbose")
|
||||||
|
|
||||||
|
var printMutex sync.Mutex
|
||||||
|
|
||||||
|
func logCallback (entry *ct.LogEntry) {
|
||||||
|
if *repo != "" {
|
||||||
|
alreadyPresent, err := ctwatch.WriteCertRepository(*repo, entry)
|
||||||
|
if err != nil {
|
||||||
|
log.Print(err)
|
||||||
|
}
|
||||||
|
if alreadyPresent {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if *script != "" {
|
||||||
|
if err := ctwatch.InvokeHookScript(*script, entry); err != nil {
|
||||||
|
log.Print(err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
printMutex.Lock()
|
||||||
|
ctwatch.DumpLogEntry(os.Stdout, entry)
|
||||||
|
fmt.Fprintf(os.Stdout, "\n")
|
||||||
|
printMutex.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func Main(logUri string, stateFile string, matcher ctwatch.Matcher) {
|
||||||
|
startIndex, err := ctwatch.ReadStateFile(stateFile)
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: Error reading state file: %s: %s\n", os.Args[0], stateFile, err)
|
||||||
|
os.Exit(3)
|
||||||
|
}
|
||||||
|
|
||||||
|
os.Setenv("LOG_URI", logUri)
|
||||||
|
|
||||||
|
logClient := client.New(logUri)
|
||||||
|
opts := ctwatch.ScannerOptions{
|
||||||
|
Matcher: matcher,
|
||||||
|
BatchSize: *batchSize,
|
||||||
|
NumWorkers: *numWorkers,
|
||||||
|
ParallelFetch: *parallelFetch,
|
||||||
|
Quiet: !*verbose,
|
||||||
|
}
|
||||||
|
scanner := ctwatch.NewScanner(logClient, opts)
|
||||||
|
|
||||||
|
endIndex, err := scanner.TreeSize()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: Error contacting log: %s: %s\n", os.Args[0], logUri, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
|
||||||
|
if startIndex != -1 {
|
||||||
|
if err := scanner.Scan(startIndex, endIndex, logCallback); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: Error scanning log: %s: %s\n", os.Args[0], logUri, err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := ctwatch.WriteStateFile(stateFile, endIndex); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "%s: Error writing state file: %s: %s\n", os.Args[0], stateFile, err)
|
||||||
|
os.Exit(3)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,45 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"bufio"
|
||||||
|
|
||||||
|
"src.agwa.name/ctwatch"
|
||||||
|
"src.agwa.name/ctwatch/cmd"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
flag.Parse()
|
||||||
|
if flag.NArg() < 2 {
|
||||||
|
fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file [domain ...]\n", os.Args[0])
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
logUri := flag.Arg(0)
|
||||||
|
stateFile := flag.Arg(1)
|
||||||
|
|
||||||
|
var domains []string
|
||||||
|
if flag.NArg() == 3 && flag.Arg(2) == "-" {
|
||||||
|
scanner := bufio.NewScanner(os.Stdin)
|
||||||
|
for scanner.Scan() {
|
||||||
|
domains = append(domains, scanner.Text())
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error reading standard input: %s\n", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
domains = flag.Args()[2:]
|
||||||
|
}
|
||||||
|
|
||||||
|
var matcher ctwatch.Matcher
|
||||||
|
if len(domains) == 0 {
|
||||||
|
matcher = ctwatch.MatchAll{}
|
||||||
|
} else {
|
||||||
|
matcher = ctwatch.NewDomainMatcher(domains)
|
||||||
|
}
|
||||||
|
|
||||||
|
cmd.Main(logUri, stateFile, matcher)
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,41 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"flag"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/google/certificate-transparency/go"
|
||||||
|
"github.com/google/certificate-transparency/go/x509"
|
||||||
|
|
||||||
|
"src.agwa.name/ctwatch/cmd"
|
||||||
|
)
|
||||||
|
|
||||||
|
type sha1Matcher struct { }
|
||||||
|
|
||||||
|
func (m sha1Matcher) CertificateMatches(c *x509.Certificate) bool {
|
||||||
|
return c.NotBefore.After(time.Date(2016, time.January, 1, 0, 0, 0, 0, time.UTC)) &&
|
||||||
|
(c.SignatureAlgorithm == x509.SHA1WithRSA ||
|
||||||
|
c.SignatureAlgorithm == x509.MD5WithRSA ||
|
||||||
|
c.SignatureAlgorithm == x509.MD2WithRSA ||
|
||||||
|
c.SignatureAlgorithm == x509.DSAWithSHA1 ||
|
||||||
|
c.SignatureAlgorithm == x509.ECDSAWithSHA1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m sha1Matcher) PrecertificateMatches(pc *ct.Precertificate) bool {
|
||||||
|
return m.CertificateMatches(&pc.TBSCertificate)
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
flag.Parse()
|
||||||
|
if flag.NArg() != 2 {
|
||||||
|
fmt.Fprintf(os.Stderr, "Usage: %s [flags] log_uri state_file\n", os.Args[0])
|
||||||
|
os.Exit(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
logUri := flag.Arg(0)
|
||||||
|
stateFile := flag.Arg(1)
|
||||||
|
|
||||||
|
cmd.Main(logUri, stateFile, &sha1Matcher{})
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,267 @@
|
||||||
|
package ctwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"time"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"bytes"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"math/big"
|
||||||
|
"path/filepath"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/pem"
|
||||||
|
|
||||||
|
"github.com/google/certificate-transparency/go"
|
||||||
|
"github.com/google/certificate-transparency/go/x509"
|
||||||
|
"github.com/google/certificate-transparency/go/x509/pkix"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ReadStateFile (path string) (int64, error) {
|
||||||
|
content, err := ioutil.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsNotExist(err) {
|
||||||
|
return -1, nil
|
||||||
|
}
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
startIndex, err := strconv.ParseInt(strings.TrimSpace(string(content)), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return -1, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return startIndex, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteStateFile (path string, endIndex int64) error {
|
||||||
|
return ioutil.WriteFile(path, []byte(strconv.FormatInt(endIndex, 10) + "\n"), 0666)
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendDnArray (buf *bytes.Buffer, code string, values []string) {
|
||||||
|
for _, value := range values {
|
||||||
|
if buf.Len() != 0 {
|
||||||
|
buf.WriteString(", ")
|
||||||
|
}
|
||||||
|
buf.WriteString(code)
|
||||||
|
buf.WriteString("=")
|
||||||
|
buf.WriteString(value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func appendDnValue (buf *bytes.Buffer, code string, value string) {
|
||||||
|
if value != "" {
|
||||||
|
appendDnArray(buf, code, []string{value})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatDN (name pkix.Name) (string) {
|
||||||
|
// C=US, ST=UT, L=Salt Lake City, O=The USERTRUST Network, OU=http://www.usertrust.com, CN=UTN-USERFirst-Hardware
|
||||||
|
var buf bytes.Buffer
|
||||||
|
appendDnArray(&buf, "C", name.Country)
|
||||||
|
appendDnArray(&buf, "ST", name.Province)
|
||||||
|
appendDnArray(&buf, "L", name.Locality)
|
||||||
|
appendDnArray(&buf, "O", name.Organization)
|
||||||
|
appendDnArray(&buf, "OU", name.OrganizationalUnit)
|
||||||
|
appendDnValue(&buf, "CN", name.CommonName)
|
||||||
|
return buf.String()
|
||||||
|
}
|
||||||
|
|
||||||
|
func allDNSNames (cert *x509.Certificate) []string {
|
||||||
|
dnsNames := []string{}
|
||||||
|
|
||||||
|
if cert.Subject.CommonName != "" {
|
||||||
|
dnsNames = append(dnsNames, cert.Subject.CommonName)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, dnsName := range cert.DNSNames {
|
||||||
|
if dnsName != cert.Subject.CommonName {
|
||||||
|
dnsNames = append(dnsNames, dnsName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return dnsNames
|
||||||
|
}
|
||||||
|
|
||||||
|
func getRoot (chain []ct.ASN1Cert) *x509.Certificate {
|
||||||
|
if len(chain) > 0 {
|
||||||
|
root, err := x509.ParseCertificate(chain[len(chain)-1])
|
||||||
|
if err == nil {
|
||||||
|
return root
|
||||||
|
}
|
||||||
|
log.Printf("Failed to parse root certificate: %s", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSubjectOrganization (cert *x509.Certificate) string {
|
||||||
|
if cert != nil && len(cert.Subject.Organization) > 0 {
|
||||||
|
return cert.Subject.Organization[0]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatSerial (serial *big.Int) string {
|
||||||
|
if serial != nil {
|
||||||
|
return fmt.Sprintf("%x", serial)
|
||||||
|
} else {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func sha256hex (data []byte) string {
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
return hex.EncodeToString(sum[:])
|
||||||
|
}
|
||||||
|
|
||||||
|
func getRaw (entry *ct.LogEntry) []byte {
|
||||||
|
if entry.Precert != nil {
|
||||||
|
return entry.Precert.Raw
|
||||||
|
} else if entry.X509Cert != nil {
|
||||||
|
return entry.X509Cert.Raw
|
||||||
|
} else {
|
||||||
|
panic("getRaw: entry is neither precert nor x509")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type certInfo struct {
|
||||||
|
IsPrecert bool
|
||||||
|
RootOrg string
|
||||||
|
SubjectDn string
|
||||||
|
IssuerDn string
|
||||||
|
DnsNames []string
|
||||||
|
Serial string
|
||||||
|
PubkeyHash string
|
||||||
|
Fingerprint string
|
||||||
|
NotBefore time.Time
|
||||||
|
NotAfter time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func makeCertInfo (entry *ct.LogEntry) certInfo {
|
||||||
|
var isPrecert bool
|
||||||
|
var cert *x509.Certificate
|
||||||
|
|
||||||
|
if entry.Precert != nil {
|
||||||
|
isPrecert = true
|
||||||
|
cert = &entry.Precert.TBSCertificate
|
||||||
|
} else if entry.X509Cert != nil {
|
||||||
|
isPrecert = false
|
||||||
|
cert = entry.X509Cert
|
||||||
|
} else {
|
||||||
|
panic("makeCertInfo: entry is neither precert nor x509")
|
||||||
|
}
|
||||||
|
return certInfo {
|
||||||
|
IsPrecert: isPrecert,
|
||||||
|
RootOrg: getSubjectOrganization(getRoot(entry.Chain)),
|
||||||
|
SubjectDn: formatDN(cert.Subject),
|
||||||
|
IssuerDn: formatDN(cert.Issuer),
|
||||||
|
DnsNames: allDNSNames(cert),
|
||||||
|
Serial: formatSerial(cert.SerialNumber),
|
||||||
|
PubkeyHash: sha256hex(cert.RawSubjectPublicKeyInfo),
|
||||||
|
Fingerprint: sha256hex(getRaw(entry)),
|
||||||
|
NotBefore: cert.NotBefore,
|
||||||
|
NotAfter: cert.NotAfter,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info *certInfo) TypeString () string {
|
||||||
|
if info.IsPrecert {
|
||||||
|
return "precert"
|
||||||
|
} else {
|
||||||
|
return "cert"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (info *certInfo) TypeFriendlyString () string {
|
||||||
|
if info.IsPrecert {
|
||||||
|
return "Pre-certificate"
|
||||||
|
} else {
|
||||||
|
return "Certificate"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func DumpLogEntry (out io.Writer, entry *ct.LogEntry) {
|
||||||
|
info := makeCertInfo(entry)
|
||||||
|
|
||||||
|
fmt.Fprintf(out, "%d:\n", entry.Index)
|
||||||
|
fmt.Fprintf(out, "\t Type = %s\n", info.TypeFriendlyString())
|
||||||
|
fmt.Fprintf(out, "\t DNS Names = %v\n", info.DnsNames)
|
||||||
|
fmt.Fprintf(out, "\t Pubkey = %s\n", info.PubkeyHash)
|
||||||
|
fmt.Fprintf(out, "\t Fingerprint = %s\n", info.Fingerprint)
|
||||||
|
fmt.Fprintf(out, "\t Subject = %s\n", info.SubjectDn)
|
||||||
|
fmt.Fprintf(out, "\t Issuer = %s\n", info.IssuerDn)
|
||||||
|
fmt.Fprintf(out, "\tRoot Operator = %s\n", info.RootOrg)
|
||||||
|
fmt.Fprintf(out, "\t Serial = %s\n", info.Serial)
|
||||||
|
fmt.Fprintf(out, "\t Not Before = %s\n", info.NotBefore)
|
||||||
|
fmt.Fprintf(out, "\t Not After = %s\n", info.NotAfter)
|
||||||
|
}
|
||||||
|
|
||||||
|
func InvokeHookScript (command string, entry *ct.LogEntry) error {
|
||||||
|
info := makeCertInfo(entry)
|
||||||
|
|
||||||
|
cmd := exec.Command(command)
|
||||||
|
cmd.Env = append(os.Environ(),
|
||||||
|
"LOG_INDEX=" + strconv.FormatInt(entry.Index, 10),
|
||||||
|
"CERT_TYPE=" + info.TypeString(),
|
||||||
|
"SUBJECT_DN=" + info.SubjectDn,
|
||||||
|
"ISSUER_DN=" + info.IssuerDn,
|
||||||
|
"DNS_NAMES=" + strings.Join(info.DnsNames, ","),
|
||||||
|
"SERIAL=" + info.Serial,
|
||||||
|
"PUBKEY_HASH=" + info.PubkeyHash,
|
||||||
|
"FINGERPRINT=" + info.Fingerprint,
|
||||||
|
"NOT_BEFORE=" + strconv.FormatInt(info.NotBefore.Unix(), 10),
|
||||||
|
"NOT_AFTER=" + strconv.FormatInt(info.NotAfter.Unix(), 10))
|
||||||
|
stderrBuffer := bytes.Buffer{}
|
||||||
|
cmd.Stderr = &stderrBuffer
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
if _, isExitError := err.(*exec.ExitError); isExitError {
|
||||||
|
fmt.Errorf("Script failed: %s: %s", command, strings.TrimSpace(stderrBuffer.String()))
|
||||||
|
} else {
|
||||||
|
fmt.Errorf("Failed to execute script: %s: %s", command, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func WriteCertRepository (repoPath string, entry *ct.LogEntry) (bool, error) {
|
||||||
|
fingerprint := sha256hex(getRaw(entry))
|
||||||
|
prefixPath := filepath.Join(repoPath, fingerprint[0:2])
|
||||||
|
var filenameSuffix string
|
||||||
|
if entry.Precert != nil {
|
||||||
|
filenameSuffix = ".precert.pem"
|
||||||
|
} else if entry.X509Cert != nil {
|
||||||
|
filenameSuffix = ".cert.pem"
|
||||||
|
}
|
||||||
|
if err := os.Mkdir(prefixPath, 0777); err != nil && !os.IsExist(err) {
|
||||||
|
return false, fmt.Errorf("Failed to create prefix directory %s: %s", prefixPath, err)
|
||||||
|
}
|
||||||
|
path := filepath.Join(prefixPath, fingerprint + filenameSuffix)
|
||||||
|
file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666)
|
||||||
|
if err != nil {
|
||||||
|
if os.IsExist(err) {
|
||||||
|
return true, nil
|
||||||
|
} else {
|
||||||
|
return false, fmt.Errorf("Failed to open %s for writing: %s", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := pem.Encode(file, &pem.Block{Type: "CERTIFICATE", Bytes: getRaw(entry)}); err != nil {
|
||||||
|
file.Close()
|
||||||
|
return false, fmt.Errorf("Error writing to %s: %s", path, err)
|
||||||
|
}
|
||||||
|
for _, chainCert := range entry.Chain {
|
||||||
|
if err := pem.Encode(file, &pem.Block{Type: "CERTIFICATE", Bytes: chainCert}); err != nil {
|
||||||
|
file.Close()
|
||||||
|
return false, fmt.Errorf("Error writing to %s: %s", path, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := file.Close(); err != nil {
|
||||||
|
return false, fmt.Errorf("Error writing to %s: %s", path, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,370 @@
|
||||||
|
package ctwatch
|
||||||
|
|
||||||
|
import (
|
||||||
|
"container/list"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/google/certificate-transparency/go"
|
||||||
|
"github.com/google/certificate-transparency/go/client"
|
||||||
|
"github.com/google/certificate-transparency/go/x509"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Clients wishing to implement their own Matchers should implement this interface:
|
||||||
|
type Matcher interface {
|
||||||
|
// CertificateMatches is called by the scanner for each X509 Certificate found in the log.
|
||||||
|
// The implementation should return |true| if the passed Certificate is interesting, and |false| otherwise.
|
||||||
|
CertificateMatches(*x509.Certificate) bool
|
||||||
|
|
||||||
|
// PrecertificateMatches is called by the scanner for each CT Precertificate found in the log.
|
||||||
|
// The implementation should return |true| if the passed Precertificate is interesting, and |false| otherwise.
|
||||||
|
PrecertificateMatches(*ct.Precertificate) bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// MatchAll is a Matcher which will match every possible Certificate and Precertificate.
|
||||||
|
type MatchAll struct{}
|
||||||
|
|
||||||
|
func (m MatchAll) CertificateMatches(_ *x509.Certificate) bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m MatchAll) PrecertificateMatches(_ *ct.Precertificate) bool {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
type DomainMatcher struct {
|
||||||
|
domains []string
|
||||||
|
domainSuffixes []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewDomainMatcher (domains []string) DomainMatcher {
|
||||||
|
m := DomainMatcher{}
|
||||||
|
for _, domain := range domains {
|
||||||
|
m.domains = append(m.domains, strings.ToLower(domain))
|
||||||
|
m.domainSuffixes = append(m.domainSuffixes, "." + strings.ToLower(domain))
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DomainMatcher) dnsNameMatches (dnsName string) bool {
|
||||||
|
dnsNameLower := strings.ToLower(dnsName)
|
||||||
|
for _, domain := range m.domains {
|
||||||
|
if dnsNameLower == domain {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, domainSuffix := range m.domainSuffixes {
|
||||||
|
if strings.HasSuffix(dnsNameLower, domainSuffix) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DomainMatcher) CertificateMatches(c *x509.Certificate) bool {
|
||||||
|
if m.dnsNameMatches(c.Subject.CommonName) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
for _, dnsName := range c.DNSNames {
|
||||||
|
if m.dnsNameMatches(dnsName) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m DomainMatcher) PrecertificateMatches(pc *ct.Precertificate) bool {
|
||||||
|
return m.CertificateMatches(&pc.TBSCertificate)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
// ScannerOptions holds configuration options for the Scanner
|
||||||
|
type ScannerOptions struct {
|
||||||
|
// Custom matcher for x509 Certificates, functor will be called for each
|
||||||
|
// Certificate found during scanning.
|
||||||
|
Matcher Matcher
|
||||||
|
|
||||||
|
// Number of entries to request in one batch from the Log
|
||||||
|
BatchSize int
|
||||||
|
|
||||||
|
// Number of concurrent matchers to run
|
||||||
|
NumWorkers int
|
||||||
|
|
||||||
|
// Number of concurrent fethers to run
|
||||||
|
ParallelFetch int
|
||||||
|
|
||||||
|
// Don't print any status messages to stdout
|
||||||
|
Quiet bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Creates a new ScannerOptions struct with sensible defaults
|
||||||
|
func DefaultScannerOptions() *ScannerOptions {
|
||||||
|
return &ScannerOptions{
|
||||||
|
Matcher: &MatchAll{},
|
||||||
|
BatchSize: 1000,
|
||||||
|
NumWorkers: 1,
|
||||||
|
ParallelFetch: 1,
|
||||||
|
Quiet: false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scanner is a tool to scan all the entries in a CT Log.
|
||||||
|
type Scanner struct {
|
||||||
|
// Client used to talk to the CT log instance
|
||||||
|
logClient *client.LogClient
|
||||||
|
|
||||||
|
// Configuration options for this Scanner instance
|
||||||
|
opts ScannerOptions
|
||||||
|
|
||||||
|
// Size of tree at end of scan
|
||||||
|
latestTreeSize int64
|
||||||
|
|
||||||
|
// Stats
|
||||||
|
certsProcessed int64
|
||||||
|
unparsableEntries int64
|
||||||
|
entriesWithNonFatalErrors int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// matcherJob represents the context for an individual matcher job.
|
||||||
|
type matcherJob struct {
|
||||||
|
// The log entry returned by the log server
|
||||||
|
entry ct.LogEntry
|
||||||
|
// The index of the entry containing the LeafInput in the log
|
||||||
|
index int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// fetchRange represents a range of certs to fetch from a CT log
|
||||||
|
type fetchRange struct {
|
||||||
|
start int64
|
||||||
|
end int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// Takes the error returned by either x509.ParseCertificate() or
|
||||||
|
// x509.ParseTBSCertificate() and determines if it's non-fatal or otherwise.
|
||||||
|
// In the case of non-fatal errors, the error will be logged,
|
||||||
|
// entriesWithNonFatalErrors will be incremented, and the return value will be
|
||||||
|
// nil.
|
||||||
|
// Fatal errors will be logged, unparsableEntires will be incremented, and the
|
||||||
|
// fatal error itself will be returned.
|
||||||
|
// When |err| is nil, this method does nothing.
|
||||||
|
func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, index int64) error {
|
||||||
|
if err == nil {
|
||||||
|
// No error to handle
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch err.(type) {
|
||||||
|
case x509.NonFatalErrors:
|
||||||
|
s.entriesWithNonFatalErrors++
|
||||||
|
// We'll make a note, but continue.
|
||||||
|
s.Warn(fmt.Sprintf("Non-fatal error in %+v at index %d: %s", entryType, index, err.Error()))
|
||||||
|
default:
|
||||||
|
s.unparsableEntries++
|
||||||
|
s.Warn(fmt.Sprintf("Failed to parse in %+v at index %d : %s", entryType, index, err.Error()))
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Processes the given |entry| in the specified log.
|
||||||
|
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry)) {
|
||||||
|
atomic.AddInt64(&s.certsProcessed, 1)
|
||||||
|
switch entry.Leaf.TimestampedEntry.EntryType {
|
||||||
|
case ct.X509LogEntryType:
|
||||||
|
cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
|
||||||
|
if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
|
||||||
|
// We hit an unparseable entry, already logged inside handleParseEntryError()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if s.opts.Matcher.CertificateMatches(cert) {
|
||||||
|
entry.X509Cert = cert
|
||||||
|
foundCert(&entry)
|
||||||
|
}
|
||||||
|
case ct.PrecertLogEntryType:
|
||||||
|
c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
|
||||||
|
if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
|
||||||
|
// We hit an unparseable entry, already logged inside handleParseEntryError()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
precert := &ct.Precertificate{
|
||||||
|
Raw: entry.Chain[0],
|
||||||
|
TBSCertificate: *c,
|
||||||
|
IssuerKeyHash: entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash,
|
||||||
|
}
|
||||||
|
if s.opts.Matcher.PrecertificateMatches(precert) {
|
||||||
|
entry.Precert = precert
|
||||||
|
foundCert(&entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Worker function to match certs.
|
||||||
|
// Accepts MatcherJobs over the |entries| channel, and processes them.
|
||||||
|
// Returns true over the |done| channel when the |entries| channel is closed.
|
||||||
|
func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry), wg *sync.WaitGroup) {
|
||||||
|
for e := range entries {
|
||||||
|
s.processEntry(e.entry, foundCert)
|
||||||
|
}
|
||||||
|
s.Log(fmt.Sprintf("Matcher %d finished", id))
|
||||||
|
wg.Done()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Worker function for fetcher jobs.
|
||||||
|
// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is
|
||||||
|
// successful sends the individual LeafInputs out (as MatcherJobs) into the
|
||||||
|
// |entries| channel for the matchers to chew on.
|
||||||
|
// Will retry failed attempts to retrieve ranges indefinitely.
|
||||||
|
// Sends true over the |done| channel when the |ranges| channel is closed.
|
||||||
|
func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- matcherJob, wg *sync.WaitGroup) {
|
||||||
|
for r := range ranges {
|
||||||
|
success := false
|
||||||
|
// TODO(alcutter): give up after a while:
|
||||||
|
for !success {
|
||||||
|
s.Log(fmt.Sprintf("Fetching entries %d to %d", r.start, r.end))
|
||||||
|
logEntries, err := s.logClient.GetEntries(r.start, r.end)
|
||||||
|
if err != nil {
|
||||||
|
s.Warn(fmt.Sprintf("Problem fetching from log: %s", err.Error()))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, logEntry := range logEntries {
|
||||||
|
logEntry.Index = r.start
|
||||||
|
entries <- matcherJob{logEntry, r.start}
|
||||||
|
r.start++
|
||||||
|
}
|
||||||
|
if r.start > r.end {
|
||||||
|
// Only complete if we actually got all the leaves we were
|
||||||
|
// expecting -- Logs MAY return fewer than the number of
|
||||||
|
// leaves requested.
|
||||||
|
success = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s.Log(fmt.Sprintf("Fetcher %d finished", id))
|
||||||
|
wg.Done()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns the smaller of |a| and |b|
|
||||||
|
func min(a int64, b int64) int64 {
|
||||||
|
if a < b {
|
||||||
|
return a
|
||||||
|
} else {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Returns the larger of |a| and |b|
|
||||||
|
func max(a int64, b int64) int64 {
|
||||||
|
if a > b {
|
||||||
|
return a
|
||||||
|
} else {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pretty prints the passed in number of |seconds| into a more human readable
|
||||||
|
// string.
|
||||||
|
func humanTime(seconds int) string {
|
||||||
|
nanos := time.Duration(seconds) * time.Second
|
||||||
|
hours := int(nanos / (time.Hour))
|
||||||
|
nanos %= time.Hour
|
||||||
|
minutes := int(nanos / time.Minute)
|
||||||
|
nanos %= time.Minute
|
||||||
|
seconds = int(nanos / time.Second)
|
||||||
|
s := ""
|
||||||
|
if hours > 0 {
|
||||||
|
s += fmt.Sprintf("%d hours ", hours)
|
||||||
|
}
|
||||||
|
if minutes > 0 {
|
||||||
|
s += fmt.Sprintf("%d minutes ", minutes)
|
||||||
|
}
|
||||||
|
if seconds > 0 {
|
||||||
|
s += fmt.Sprintf("%d seconds ", seconds)
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Scanner) Log(msg string) {
|
||||||
|
if !s.opts.Quiet {
|
||||||
|
log.Print(msg)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Scanner) Warn(msg string) {
|
||||||
|
log.Print(msg)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scanner) TreeSize() (int64, error) {
|
||||||
|
latestSth, err := s.logClient.GetSTH()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
return int64(latestSth.TreeSize), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Scanner) Scan(startIndex int64, endIndex int64, foundCert func(*ct.LogEntry)) error {
|
||||||
|
s.Log("Starting up...\n")
|
||||||
|
|
||||||
|
s.certsProcessed = 0
|
||||||
|
s.unparsableEntries = 0
|
||||||
|
s.entriesWithNonFatalErrors = 0
|
||||||
|
ticker := time.NewTicker(time.Second)
|
||||||
|
startTime := time.Now()
|
||||||
|
fetches := make(chan fetchRange, 1000)
|
||||||
|
jobs := make(chan matcherJob, 100000)
|
||||||
|
go func() {
|
||||||
|
for range ticker.C {
|
||||||
|
throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds()
|
||||||
|
remainingCerts := int64(endIndex) - int64(startIndex) - s.certsProcessed
|
||||||
|
remainingSeconds := int(float64(remainingCerts) / throughput)
|
||||||
|
remainingString := humanTime(remainingSeconds)
|
||||||
|
s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s\n", s.certsProcessed,
|
||||||
|
startIndex+int64(s.certsProcessed), throughput, remainingString))
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
var ranges list.List
|
||||||
|
for start := startIndex; start < int64(endIndex); {
|
||||||
|
end := min(start+int64(s.opts.BatchSize), int64(endIndex)) - 1
|
||||||
|
ranges.PushBack(fetchRange{start, end})
|
||||||
|
start = end + 1
|
||||||
|
}
|
||||||
|
var fetcherWG sync.WaitGroup
|
||||||
|
var matcherWG sync.WaitGroup
|
||||||
|
// Start matcher workers
|
||||||
|
for w := 0; w < s.opts.NumWorkers; w++ {
|
||||||
|
matcherWG.Add(1)
|
||||||
|
go s.matcherJob(w, jobs, foundCert, &matcherWG)
|
||||||
|
}
|
||||||
|
// Start fetcher workers
|
||||||
|
for w := 0; w < s.opts.ParallelFetch; w++ {
|
||||||
|
fetcherWG.Add(1)
|
||||||
|
go s.fetcherJob(w, fetches, jobs, &fetcherWG)
|
||||||
|
}
|
||||||
|
for r := ranges.Front(); r != nil; r = r.Next() {
|
||||||
|
fetches <- r.Value.(fetchRange)
|
||||||
|
}
|
||||||
|
close(fetches)
|
||||||
|
fetcherWG.Wait()
|
||||||
|
close(jobs)
|
||||||
|
matcherWG.Wait()
|
||||||
|
s.Log(fmt.Sprintf("Completed %d certs in %s", s.certsProcessed, humanTime(int(time.Since(startTime).Seconds()))))
|
||||||
|
s.Log(fmt.Sprintf("%d unparsable entries, %d non-fatal errors", s.unparsableEntries, s.entriesWithNonFatalErrors))
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Creates a new Scanner instance using |client| to talk to the log, and taking
|
||||||
|
// configuration options from |opts|.
|
||||||
|
func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner {
|
||||||
|
var scanner Scanner
|
||||||
|
scanner.logClient = client
|
||||||
|
// Set a default match-everything regex if none was provided:
|
||||||
|
if opts.Matcher == nil {
|
||||||
|
opts.Matcher = &MatchAll{}
|
||||||
|
}
|
||||||
|
scanner.opts = opts
|
||||||
|
return &scanner
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue