2016-02-05 03:45:37 +01:00
package cmd
import (
"flag"
"fmt"
"log"
"os"
2016-02-17 23:54:25 +01:00
"bytes"
2016-02-05 05:16:25 +01:00
"os/user"
2016-02-18 01:03:49 +01:00
"encoding/json"
2016-02-05 03:45:37 +01:00
"sync"
2016-02-05 05:16:25 +01:00
"strings"
"path/filepath"
2016-02-18 21:40:21 +01:00
"time"
"strconv"
2016-02-05 03:45:37 +01:00
"src.agwa.name/ctwatch"
2016-02-18 21:40:21 +01:00
"src.agwa.name/ctwatch/ct"
2016-02-05 03:45:37 +01:00
)
var batchSize = flag . Int ( "batch_size" , 1000 , "Max number of entries to request at per call to get-entries" )
var numWorkers = flag . Int ( "num_workers" , 2 , "Number of concurrent matchers" )
var parallelFetch = flag . Int ( "parallel_fetch" , 2 , "Number of concurrent GetEntries fetches" )
var script = flag . String ( "script" , "" , "Script to execute when a matching certificate is found" )
2016-02-18 01:03:49 +01:00
var logsFilename = flag . String ( "logs" , "" , "JSON file containing log URLs" )
2016-02-23 00:14:17 +01:00
var underwater = flag . Bool ( "underwater" , false , "Monitor certificates from distrusted CAs instead of trusted CAs" )
2016-02-05 05:16:25 +01:00
var noSave = flag . Bool ( "no_save" , false , "Do not save a copy of matching certificates" )
2016-02-05 03:45:37 +01:00
var verbose = flag . Bool ( "verbose" , false , "Be verbose" )
2016-02-17 23:54:25 +01:00
var allTime = flag . Bool ( "all_time" , false , "Scan certs from all time, not just since last scan" )
2016-02-05 05:16:25 +01:00
var stateDir string
2016-02-05 03:45:37 +01:00
var printMutex sync . Mutex
2016-02-05 05:16:25 +01:00
func isRoot ( ) bool {
return os . Geteuid ( ) == 0
}
func homedir ( ) string {
home := os . Getenv ( "HOME" )
if home != "" {
return home
}
user , err := user . Current ( )
if err == nil {
return user . HomeDir
}
panic ( "Unable to determine home directory" )
}
func DefaultStateDir ( programName string ) string {
if isRoot ( ) {
return filepath . Join ( "/var/lib" , programName )
} else {
return filepath . Join ( homedir ( ) , "." + programName )
}
}
2016-02-09 19:28:52 +01:00
func LogEntry ( info * ctwatch . EntryInfo ) {
2016-02-05 05:16:25 +01:00
if ! * noSave {
2016-02-05 17:20:12 +01:00
var alreadyPresent bool
var err error
2016-02-09 19:28:52 +01:00
alreadyPresent , info . Filename , err = ctwatch . WriteCertRepository ( filepath . Join ( stateDir , "certs" ) , info . Entry )
2016-02-05 03:45:37 +01:00
if err != nil {
log . Print ( err )
}
if alreadyPresent {
return
}
}
if * script != "" {
2016-02-09 19:28:52 +01:00
if err := info . InvokeHookScript ( * script ) ; err != nil {
2016-02-05 03:45:37 +01:00
log . Print ( err )
}
} else {
printMutex . Lock ( )
2016-02-09 19:28:52 +01:00
info . Write ( os . Stdout )
2016-02-05 03:45:37 +01:00
fmt . Fprintf ( os . Stdout , "\n" )
printMutex . Unlock ( )
}
}
2016-02-05 05:16:25 +01:00
func defangLogUri ( logUri string ) string {
return strings . Replace ( strings . Replace ( logUri , "://" , "_" , 1 ) , "/" , "_" , - 1 )
}
2016-02-05 03:45:37 +01:00
2016-02-18 21:40:21 +01:00
func saveEvidence ( logUri string , firstSTH * ct . SignedTreeHead , secondSTH * ct . SignedTreeHead ) ( string , string , error ) {
now := strconv . FormatInt ( time . Now ( ) . Unix ( ) , 10 )
firstFilename := filepath . Join ( stateDir , "evidence" , defangLogUri ( logUri ) + ".inconsistent." + now + ".first" )
if err := ctwatch . WriteSTHFile ( firstFilename , firstSTH ) ; err != nil {
return "" , "" , err
}
secondFilename := filepath . Join ( stateDir , "evidence" , defangLogUri ( logUri ) + ".inconsistent." + now + ".second" )
if err := ctwatch . WriteSTHFile ( secondFilename , secondSTH ) ; err != nil {
return "" , "" , err
}
return firstFilename , secondFilename , nil
}
2016-02-09 19:28:52 +01:00
func Main ( argStateDir string , processCallback ctwatch . ProcessCallback ) {
2016-02-05 05:16:25 +01:00
stateDir = argStateDir
2016-02-05 03:45:37 +01:00
2016-02-18 01:03:49 +01:00
var logs [ ] ctwatch . LogInfo
2016-02-05 05:16:25 +01:00
if * logsFilename != "" {
logFile , err := os . Open ( * logsFilename )
if err != nil {
fmt . Fprintf ( os . Stderr , "%s: Error opening logs file for reading: %s: %s\n" , os . Args [ 0 ] , * logsFilename , err )
2016-02-22 23:45:50 +01:00
os . Exit ( 1 )
2016-02-05 05:16:25 +01:00
}
defer logFile . Close ( )
2016-02-18 01:03:49 +01:00
var logFileObj ctwatch . LogInfoFile
if err := json . NewDecoder ( logFile ) . Decode ( & logFileObj ) ; err != nil {
fmt . Fprintf ( os . Stderr , "%s: Error decoding logs file: %s: %s\n" , os . Args [ 0 ] , * logsFilename , err )
2016-02-22 23:45:50 +01:00
os . Exit ( 1 )
2016-02-05 05:16:25 +01:00
}
2016-02-18 01:03:49 +01:00
logs = logFileObj . Logs
} else if * underwater {
logs = ctwatch . UnderwaterLogs
2016-02-05 05:16:25 +01:00
} else {
2016-02-18 01:03:49 +01:00
logs = ctwatch . DefaultLogs
2016-02-05 03:45:37 +01:00
}
2016-02-05 05:16:25 +01:00
if err := os . Mkdir ( stateDir , 0777 ) ; err != nil && ! os . IsExist ( err ) {
fmt . Fprintf ( os . Stderr , "%s: Error creating state directory: %s: %s\n" , os . Args [ 0 ] , stateDir , err )
2016-02-22 23:45:50 +01:00
os . Exit ( 1 )
2016-02-05 03:45:37 +01:00
}
2016-02-18 21:40:21 +01:00
for _ , subdir := range [ ] string { "certs" , "sths" , "evidence" } {
2016-02-05 05:16:25 +01:00
path := filepath . Join ( stateDir , subdir )
if err := os . Mkdir ( path , 0777 ) ; err != nil && ! os . IsExist ( err ) {
fmt . Fprintf ( os . Stderr , "%s: Error creating state directory: %s: %s\n" , os . Args [ 0 ] , path , err )
2016-02-22 23:45:50 +01:00
os . Exit ( 1 )
2016-02-05 03:45:37 +01:00
}
}
2016-02-22 23:45:50 +01:00
/ *
* Exit code bits :
2016-02-22 23:58:11 +01:00
* 1 = initialization / configuration / system error
2016-02-22 23:45:50 +01:00
* 2 = usage error
* 4 = error communicating with log
* 8 = log misbehavior
* /
2016-02-05 05:16:25 +01:00
exitCode := 0
2016-02-18 01:03:49 +01:00
for _ , logInfo := range logs {
logUri := logInfo . FullURI ( )
2016-02-22 23:58:11 +01:00
log . SetPrefix ( os . Args [ 0 ] + ": " + logUri + ": " )
2016-02-18 01:03:49 +01:00
logKey , err := logInfo . ParsedPublicKey ( )
if err != nil {
2016-02-22 23:58:11 +01:00
log . Printf ( "Bad public key: %s\n" , err )
exitCode |= 1
continue
2016-02-18 01:03:49 +01:00
}
2016-02-17 23:54:25 +01:00
stateFilename := filepath . Join ( stateDir , "sths" , defangLogUri ( logUri ) )
2016-02-18 19:09:33 +01:00
prevSTH , err := ctwatch . ReadSTHFile ( stateFilename )
2016-02-05 05:16:25 +01:00
if err != nil {
2016-02-22 23:58:11 +01:00
log . Printf ( "Error reading state file: %s: %s\n" , stateFilename , err )
exitCode |= 1
continue
2016-02-05 05:16:25 +01:00
}
opts := ctwatch . ScannerOptions {
BatchSize : * batchSize ,
NumWorkers : * numWorkers ,
ParallelFetch : * parallelFetch ,
Quiet : ! * verbose ,
}
2016-02-23 00:29:52 +01:00
scanner := ctwatch . NewScanner ( logUri , logKey , & opts )
2016-02-05 05:16:25 +01:00
2016-02-17 23:54:25 +01:00
latestSTH , err := scanner . GetSTH ( )
2016-02-05 05:16:25 +01:00
if err != nil {
2016-02-22 23:58:11 +01:00
log . Printf ( "Error retrieving STH from log: %s\n" , err )
2016-02-22 23:45:50 +01:00
exitCode |= 4
2016-02-05 05:16:25 +01:00
continue
}
2016-02-22 23:58:11 +01:00
if * verbose {
if prevSTH != nil {
log . Printf ( "Existing log; scanning %d new entries since previous scan (previous size %d, previous root hash = %x)" , latestSTH . TreeSize - prevSTH . TreeSize , prevSTH . TreeSize , prevSTH . SHA256RootHash )
} else if * allTime {
log . Printf ( "new log; scanning all %d entries in the log" , latestSTH . TreeSize )
} else {
log . Printf ( "new log; not scanning existing entries because -all_time option not specified" )
}
}
2016-02-17 23:54:25 +01:00
var startIndex uint64
2016-02-20 21:04:07 +01:00
if prevSTH != nil {
2016-02-17 23:54:25 +01:00
startIndex = prevSTH . TreeSize
2016-02-20 21:04:07 +01:00
} else if * allTime {
startIndex = 0
2016-02-17 23:54:25 +01:00
} else {
startIndex = latestSTH . TreeSize
}
if latestSTH . TreeSize > startIndex {
2016-02-18 19:15:56 +01:00
var treeBuilder * ctwatch . MerkleTreeBuilder
2016-02-17 23:54:25 +01:00
if prevSTH != nil {
2016-02-18 19:23:07 +01:00
var valid bool
var err error
valid , treeBuilder , err = scanner . CheckConsistency ( prevSTH , latestSTH )
2016-02-17 23:54:25 +01:00
if err != nil {
2016-02-22 23:58:11 +01:00
log . Printf ( "Error fetching consistency proof: %s\n" , err )
2016-02-22 23:45:50 +01:00
exitCode |= 4
2016-02-17 23:54:25 +01:00
continue
}
if ! valid {
2016-02-18 21:40:21 +01:00
firstFilename , secondFilename , err := saveEvidence ( logUri , prevSTH , latestSTH )
if err != nil {
2016-02-22 23:58:11 +01:00
log . Printf ( "Consistency proof failed - the log has misbehaved! Saving evidence of misbehavior failed: %s\n" , err )
2016-02-18 21:40:21 +01:00
} else {
2016-02-22 23:58:11 +01:00
log . Printf ( "Consistency proof failed - the log has misbehaved! Evidence of misbehavior has been saved to '%s' and '%s'.\n" , firstFilename , secondFilename )
2016-02-18 21:40:21 +01:00
}
2016-02-22 23:45:50 +01:00
exitCode |= 8
2016-02-17 23:54:25 +01:00
continue
}
} else {
2016-02-18 19:15:56 +01:00
treeBuilder = & ctwatch . MerkleTreeBuilder { }
2016-02-17 23:54:25 +01:00
}
2016-02-18 19:15:56 +01:00
if err := scanner . Scan ( int64 ( startIndex ) , int64 ( latestSTH . TreeSize ) , processCallback , treeBuilder ) ; err != nil {
2016-02-22 23:58:11 +01:00
log . Printf ( "Error scanning log: %s\n" , err )
2016-02-22 23:45:50 +01:00
exitCode |= 4
2016-02-05 05:16:25 +01:00
continue
}
2016-02-17 23:54:25 +01:00
2016-02-18 19:15:56 +01:00
rootHash := treeBuilder . Finish ( )
2016-02-17 23:54:25 +01:00
if ! bytes . Equal ( rootHash , latestSTH . SHA256RootHash [ : ] ) {
2016-02-22 23:58:11 +01:00
log . Printf ( "Validation of log entries failed - calculated tree root (%x) does not match signed tree root (%s). If this error persists for an extended period, it should be construed as misbehavior by the log.\n" , rootHash , latestSTH . SHA256RootHash )
2016-02-22 23:45:50 +01:00
exitCode |= 8
2016-02-17 23:54:25 +01:00
continue
}
2016-02-05 05:16:25 +01:00
}
2016-02-22 23:58:11 +01:00
if * verbose {
log . Printf ( "final log size = %d, final root hash = %x" , latestSTH . TreeSize , latestSTH . SHA256RootHash )
}
2016-02-18 19:09:33 +01:00
if err := ctwatch . WriteSTHFile ( stateFilename , latestSTH ) ; err != nil {
2016-02-22 23:58:11 +01:00
log . Printf ( "Error writing state file: %s: %s\n" , stateFilename , err )
exitCode |= 1
continue
2016-02-05 05:16:25 +01:00
}
2016-02-05 03:45:37 +01:00
}
2016-02-05 05:16:25 +01:00
os . Exit ( exitCode )
2016-02-05 03:45:37 +01:00
}