diff --git a/ct/AUTHORS b/ct/AUTHORS deleted file mode 100644 index ac5678b..0000000 --- a/ct/AUTHORS +++ /dev/null @@ -1,24 +0,0 @@ -# This is the official list of benchmark authors for copyright purposes. -# This file is distinct from the CONTRIBUTORS files. -# See the latter for an explanation. -# -# Names should be added to this file as: -# Name or Organization -# The email address is not required for organizations. -# -# Please keep the list sorted. - -Comodo CA Limited -Ed Maste -Fiaz Hossain -Google Inc. -Jeff Trawick -Katriel Cohn-Gordon -Mark Schloesser -NORDUnet A/S -Nicholas Galbreath -Oliver Weidner -Ruslan Kovalov -Venafi, Inc. -Vladimir Rutsky -Ximin Luo diff --git a/ct/LICENSE b/ct/LICENSE deleted file mode 100644 index d645695..0000000 --- a/ct/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/ct/README b/ct/README deleted file mode 100644 index 1a3d959..0000000 --- a/ct/README +++ /dev/null @@ -1,4 +0,0 @@ -The code in this directory is based on Google's Certificiate Transparency Go library -(originally at ; -now at ). -See AUTHORS for the copyright holders, and LICENSE for the license. diff --git a/ct/client/logclient.go b/ct/client/logclient.go deleted file mode 100644 index b0d294a..0000000 --- a/ct/client/logclient.go +++ /dev/null @@ -1,416 +0,0 @@ -// Package client is a CT log client implementation and contains types and code -// for interacting with RFC6962-compliant CT Log instances. -// See http://tools.ietf.org/html/rfc6962 for details -package client - -import ( - "bytes" - "context" - "crypto/sha256" - "crypto/tls" - "encoding/base64" - "encoding/json" - "errors" - "fmt" - "io" - insecurerand "math/rand" - "net/http" - "net/url" - "strconv" - "time" - - "software.sslmate.com/src/certspotter/ct" -) - -const ( - baseRetryDelay = 1 * time.Second - maxRetryDelay = 120 * time.Second - maxRetries = 10 -) - -func isRetryableStatusCode(code int) bool { - return code/100 == 5 || code == http.StatusTooManyRequests -} - -func randomDuration(min, max time.Duration) time.Duration { - return min + time.Duration(insecurerand.Int63n(int64(max)-int64(min)+1)) -} - -func getRetryAfter(resp *http.Response) (time.Duration, bool) { - if resp == nil { - return 0, false - } - seconds, err := strconv.ParseUint(resp.Header.Get("Retry-After"), 10, 16) - if err != nil { - return 0, false - } - return time.Duration(seconds) * time.Second, true -} - -func sleep(ctx context.Context, duration time.Duration) { - timer := time.NewTimer(duration) - defer timer.Stop() - select { - case <-ctx.Done(): - case <-timer.C: - } -} - -// URI paths for CT Log endpoints -const ( - GetSTHPath = "/ct/v1/get-sth" - GetEntriesPath = "/ct/v1/get-entries" - GetSTHConsistencyPath = "/ct/v1/get-sth-consistency" - GetProofByHashPath = "/ct/v1/get-proof-by-hash" - AddChainPath = "/ct/v1/add-chain" -) - -// LogClient represents a client for a given CT Log instance -type LogClient struct { - uri string // the base URI of the log. e.g. http://ct.googleapis/pilot - httpClient *http.Client // used to interact with the log via HTTP - verifier *ct.SignatureVerifier // if non-nil, used to verify STH signatures -} - -////////////////////////////////////////////////////////////////////////////////// -// JSON structures follow. -// These represent the structures returned by the CT Log server. -////////////////////////////////////////////////////////////////////////////////// - -// getSTHResponse represents the JSON response to the get-sth CT method -type getSTHResponse struct { - TreeSize uint64 `json:"tree_size"` // Number of certs in the current tree - Timestamp uint64 `json:"timestamp"` // Time that the tree was created - SHA256RootHash []byte `json:"sha256_root_hash"` // Root hash of the tree - TreeHeadSignature []byte `json:"tree_head_signature"` // Log signature for this STH -} - -// base64LeafEntry represents a Base64 encoded leaf entry -type base64LeafEntry struct { - LeafInput []byte `json:"leaf_input"` - ExtraData []byte `json:"extra_data"` -} - -// getEntriesReponse represents the JSON response to the CT get-entries method -type getEntriesResponse struct { - Entries []base64LeafEntry `json:"entries"` // the list of returned entries -} - -// getConsistencyProofResponse represents the JSON response to the CT get-consistency-proof method -type getConsistencyProofResponse struct { - Consistency [][]byte `json:"consistency"` -} - -// getAuditProofResponse represents the JSON response to the CT get-proof-by-hash method -type getAuditProofResponse struct { - LeafIndex uint64 `json:"leaf_index"` - AuditPath [][]byte `json:"audit_path"` -} - -type addChainRequest struct { - Chain [][]byte `json:"chain"` -} - -type addChainResponse struct { - SCTVersion uint8 `json:"sct_version"` - ID []byte `json:"id"` - Timestamp uint64 `json:"timestamp"` - Extensions []byte `json:"extensions"` - Signature []byte `json:"signature"` -} - -// New constructs a new LogClient instance. -// |uri| is the base URI of the CT log instance to interact with, e.g. -// http://ct.googleapis.com/pilot -func New(uri string) *LogClient { - return NewWithVerifier(uri, nil) -} - -func NewWithVerifier(uri string, verifier *ct.SignatureVerifier) *LogClient { - var c LogClient - c.uri = uri - c.verifier = verifier - transport := &http.Transport{ - Proxy: http.ProxyFromEnvironment, - TLSHandshakeTimeout: 15 * time.Second, - ResponseHeaderTimeout: 30 * time.Second, - MaxIdleConnsPerHost: 10, - DisableKeepAlives: false, - MaxIdleConns: 100, - IdleConnTimeout: 15 * time.Second, - ExpectContinueTimeout: 1 * time.Second, - TLSClientConfig: &tls.Config{ - // We have to disable TLS certificate validation because because several logs - // (WoSign, StartCom, GDCA) use certificates that are not widely trusted. - // Since we verify that every response we receive from the log is signed - // by the log's CT public key (either directly, or indirectly via the Merkle Tree), - // TLS certificate validation is not actually necessary. (We don't want to ship - // our own trust store because that adds undesired complexity and would require - // updating should a log ever change to a different CA.) - InsecureSkipVerify: true, - }, - } - c.httpClient = &http.Client{Timeout: 60 * time.Second, Transport: transport} - return &c -} - -func (c *LogClient) fetchAndParse(ctx context.Context, uri string, respBody interface{}) error { - return c.doAndParse(ctx, "GET", uri, nil, respBody) -} - -func (c *LogClient) postAndParse(ctx context.Context, uri string, body interface{}, respBody interface{}) error { - return c.doAndParse(ctx, "POST", uri, body, respBody) -} - -func (c *LogClient) makeRequest(ctx context.Context, method string, uri string, body interface{}) (*http.Request, error) { - if body == nil { - return http.NewRequestWithContext(ctx, method, uri, nil) - } else { - bodyBytes, err := json.Marshal(body) - if err != nil { - return nil, err - } - req, err := http.NewRequestWithContext(ctx, method, uri, bytes.NewReader(bodyBytes)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - return req, nil - } -} - -func (c *LogClient) doAndParse(ctx context.Context, method string, uri string, reqBody interface{}, respBody interface{}) error { - numRetries := 0 -retry: - if ctx.Err() != nil { - return ctx.Err() - } - req, err := c.makeRequest(ctx, method, uri, reqBody) - if err != nil { - return fmt.Errorf("%s %s: error creating request: %w", method, uri, err) - } - req.Header.Set("User-Agent", "") // Don't send a User-Agent to make life harder for malicious logs - resp, err := c.httpClient.Do(req) - if err != nil { - if c.shouldRetry(ctx, numRetries, nil) { - numRetries++ - goto retry - } - return err - } - respBodyBytes, err := io.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - if c.shouldRetry(ctx, numRetries, nil) { - numRetries++ - goto retry - } - return fmt.Errorf("%s %s: error reading response: %w", method, uri, err) - } - if resp.StatusCode/100 != 2 { - if c.shouldRetry(ctx, numRetries, resp) { - numRetries++ - goto retry - } - return fmt.Errorf("%s %s: %s (%s)", method, uri, resp.Status, string(respBodyBytes)) - } - if err := json.Unmarshal(respBodyBytes, respBody); err != nil { - return fmt.Errorf("%s %s: error parsing response JSON: %w", method, uri, err) - } - return nil -} - -func (c *LogClient) shouldRetry(ctx context.Context, numRetries int, resp *http.Response) bool { - if numRetries == maxRetries { - return false - } - - if resp != nil && !isRetryableStatusCode(resp.StatusCode) { - return false - } - - var delay time.Duration - if retryAfter, hasRetryAfter := getRetryAfter(resp); hasRetryAfter { - delay = retryAfter - } else { - delay = baseRetryDelay * (1 << numRetries) - if delay > maxRetryDelay { - delay = maxRetryDelay - } - delay += randomDuration(0, delay/2) - } - - if deadline, hasDeadline := ctx.Deadline(); hasDeadline && time.Now().Add(delay).After(deadline) { - return false - } - - sleep(ctx, delay) - return true -} - -// GetSTH retrieves the current STH from the log. -// Returns a populated SignedTreeHead, or a non-nil error. -func (c *LogClient) GetSTH(ctx context.Context) (sth *ct.SignedTreeHead, err error) { - var resp getSTHResponse - if err = c.fetchAndParse(ctx, c.uri+GetSTHPath, &resp); err != nil { - return - } - sth = &ct.SignedTreeHead{ - TreeSize: resp.TreeSize, - Timestamp: resp.Timestamp, - } - - if len(resp.SHA256RootHash) != sha256.Size { - return nil, fmt.Errorf("STH returned by server has invalid sha256_root_hash (expected length %d got %d)", sha256.Size, len(resp.SHA256RootHash)) - } - copy(sth.SHA256RootHash[:], resp.SHA256RootHash) - - ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.TreeHeadSignature)) - if err != nil { - return nil, err - } - sth.TreeHeadSignature = *ds - if c.verifier != nil { - if err := c.verifier.VerifySTHSignature(*sth); err != nil { - return nil, fmt.Errorf("STH returned by server has invalid signature: %w", err) - } - } - return -} - -type GetEntriesItem struct { - LeafInput []byte `json:"leaf_input"` - ExtraData []byte `json:"extra_data"` -} - -// Retrieve the entries in the sequence [start, end] from the CT log server. -// If error is nil, at least one entry is returned, and no excess entries are returned. -// Fewer entries than requested may be returned. -func (c *LogClient) GetRawEntries(ctx context.Context, start, end uint64) ([]GetEntriesItem, error) { - if end < start { - panic("LogClient.GetRawEntries: end < start") - } - var response struct { - Entries []GetEntriesItem `json:"entries"` - } - uri := fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end) - err := c.fetchAndParse(ctx, uri, &response) - if err != nil { - return nil, err - } - if len(response.Entries) == 0 { - return nil, fmt.Errorf("GET %s: log server returned an empty get-entries response", uri) - } - if uint64(len(response.Entries)) > end-start+1 { - return nil, fmt.Errorf("GET %s: log server returned a get-entries response with extraneous entries", uri) - } - return response.Entries, nil -} - -// GetEntries attempts to retrieve the entries in the sequence [|start|, |end|] from the CT -// log server. (see section 4.6.) -// Returns a slice of LeafInputs or a non-nil error. -func (c *LogClient) GetEntries(ctx context.Context, start, end int64) ([]ct.LogEntry, error) { - if end < 0 { - return nil, errors.New("GetEntries: end should be >= 0") - } - if end < start { - return nil, errors.New("GetEntries: start should be <= end") - } - var resp getEntriesResponse - err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?start=%d&end=%d", c.uri, GetEntriesPath, start, end), &resp) - if err != nil { - return nil, err - } - entries := make([]ct.LogEntry, len(resp.Entries)) - for index, entry := range resp.Entries { - leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewBuffer(entry.LeafInput)) - if err != nil { - return nil, fmt.Errorf("Reading Merkle Tree Leaf at index %d failed: %s", start+int64(index), err) - } - entries[index].LeafBytes = entry.LeafInput - entries[index].Leaf = *leaf - - var chain []ct.ASN1Cert - switch leaf.TimestampedEntry.EntryType { - case ct.X509LogEntryType: - chain, err = ct.UnmarshalX509ChainArray(entry.ExtraData) - - case ct.PrecertLogEntryType: - chain, err = ct.UnmarshalPrecertChainArray(entry.ExtraData) - - default: - return nil, fmt.Errorf("Unknown entry type at index %d: %v", start+int64(index), leaf.TimestampedEntry.EntryType) - } - if err != nil { - return nil, fmt.Errorf("Parsing entry of type %d at index %d failed: %s", leaf.TimestampedEntry.EntryType, start+int64(index), err) - } - entries[index].Chain = chain - entries[index].Index = start + int64(index) - } - return entries, nil -} - -// GetConsistencyProof retrieves a Merkle Consistency Proof between two STHs (|first| and |second|) -// from the log. Returns a slice of MerkleTreeNodes (a ct.ConsistencyProof) or a non-nil error. -func (c *LogClient) GetConsistencyProof(ctx context.Context, first, second int64) (ct.ConsistencyProof, error) { - if second < 0 { - return nil, errors.New("GetConsistencyProof: second should be >= 0") - } - if second < first { - return nil, errors.New("GetConsistencyProof: first should be <= second") - } - var resp getConsistencyProofResponse - err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?first=%d&second=%d", c.uri, GetSTHConsistencyPath, first, second), &resp) - if err != nil { - return nil, err - } - nodes := make([]ct.MerkleTreeNode, len(resp.Consistency)) - for index, nodeBytes := range resp.Consistency { - nodes[index] = nodeBytes - } - return nodes, nil -} - -// GetAuditProof retrieves a Merkle Audit Proof (aka Inclusion Proof) for the given -// |hash| based on the STH at |treeSize| from the log. Returns a slice of MerkleTreeNodes -// and the index of the leaf. -func (c *LogClient) GetAuditProof(ctx context.Context, hash ct.MerkleTreeNode, treeSize uint64) (ct.AuditPath, uint64, error) { - var resp getAuditProofResponse - err := c.fetchAndParse(ctx, fmt.Sprintf("%s%s?hash=%s&tree_size=%d", c.uri, GetProofByHashPath, url.QueryEscape(base64.StdEncoding.EncodeToString(hash)), treeSize), &resp) - if err != nil { - return nil, 0, err - } - path := make([]ct.MerkleTreeNode, len(resp.AuditPath)) - for index, nodeBytes := range resp.AuditPath { - path[index] = nodeBytes - } - return path, resp.LeafIndex, nil -} - -func (c *LogClient) AddChain(ctx context.Context, chain [][]byte) (*ct.SignedCertificateTimestamp, error) { - req := addChainRequest{Chain: chain} - - var resp addChainResponse - if err := c.postAndParse(ctx, c.uri+AddChainPath, &req, &resp); err != nil { - return nil, err - } - - sct := &ct.SignedCertificateTimestamp{ - SCTVersion: ct.Version(resp.SCTVersion), - Timestamp: resp.Timestamp, - Extensions: resp.Extensions, - } - - if len(resp.ID) != sha256.Size { - return nil, fmt.Errorf("SCT returned by server has invalid id (expected length %d got %d)", sha256.Size, len(resp.ID)) - } - copy(sct.LogID[:], resp.ID) - - ds, err := ct.UnmarshalDigitallySigned(bytes.NewReader(resp.Signature)) - if err != nil { - return nil, err - } - sct.Signature = *ds - return sct, nil -} diff --git a/ct/serialization.go b/ct/serialization.go deleted file mode 100644 index 793a07f..0000000 --- a/ct/serialization.go +++ /dev/null @@ -1,462 +0,0 @@ -package ct - -import ( - "bytes" - "container/list" - "crypto" - "encoding/binary" - "errors" - "fmt" - "io" -) - -// Variable size structure prefix-header byte lengths -const ( - CertificateLengthBytes = 3 - PreCertificateLengthBytes = 3 - ExtensionsLengthBytes = 2 - CertificateChainLengthBytes = 3 - SignatureLengthBytes = 2 -) - -// Max lengths -const ( - MaxCertificateLength = (1 << 24) - 1 - MaxExtensionsLength = (1 << 16) - 1 -) - -func writeUint(w io.Writer, value uint64, numBytes int) error { - buf := make([]uint8, numBytes) - for i := 0; i < numBytes; i++ { - buf[numBytes-i-1] = uint8(value & 0xff) - value >>= 8 - } - if value != 0 { - return errors.New("numBytes was insufficiently large to represent value") - } - if _, err := w.Write(buf); err != nil { - return err - } - return nil -} - -func writeVarBytes(w io.Writer, value []byte, numLenBytes int) error { - if err := writeUint(w, uint64(len(value)), numLenBytes); err != nil { - return err - } - if _, err := w.Write(value); err != nil { - return err - } - return nil -} - -func readUint(r io.Reader, numBytes int) (uint64, error) { - var l uint64 - for i := 0; i < numBytes; i++ { - l <<= 8 - var t uint8 - if err := binary.Read(r, binary.BigEndian, &t); err != nil { - return 0, err - } - l |= uint64(t) - } - return l, nil -} - -// Reads a variable length array of bytes from |r|. |numLenBytes| specifies the -// number of (BigEndian) prefix-bytes which contain the length of the actual -// array data bytes that follow. -// Allocates an array to hold the contents and returns a slice view into it if -// the read was successful, or an error otherwise. -func readVarBytes(r io.Reader, numLenBytes int) ([]byte, error) { - switch { - case numLenBytes > 8: - return nil, fmt.Errorf("numLenBytes too large (%d)", numLenBytes) - case numLenBytes == 0: - return nil, errors.New("numLenBytes should be > 0") - } - l, err := readUint(r, numLenBytes) - if err != nil { - return nil, err - } - data := make([]byte, l) - if n, err := io.ReadFull(r, data); err != nil { - if err == io.EOF || err == io.ErrUnexpectedEOF { - return nil, fmt.Errorf("short read: expected %d but got %d", l, n) - } - return nil, err - } - return data, nil -} - -// Reads a list of ASN1Cert types from |r| -func readASN1CertList(r io.Reader, totalLenBytes int, elementLenBytes int) ([]ASN1Cert, error) { - listBytes, err := readVarBytes(r, totalLenBytes) - if err != nil { - return []ASN1Cert{}, err - } - list := list.New() - listReader := bytes.NewReader(listBytes) - var entry []byte - for err == nil { - entry, err = readVarBytes(listReader, elementLenBytes) - if err != nil { - if err != io.EOF { - return []ASN1Cert{}, err - } - } else { - list.PushBack(entry) - } - } - ret := make([]ASN1Cert, list.Len()) - i := 0 - for e := list.Front(); e != nil; e = e.Next() { - ret[i] = e.Value.([]byte) - i++ - } - return ret, nil -} - -// ReadTimestampedEntryInto parses the byte-stream representation of a -// TimestampedEntry from |r| and populates the struct |t| with the data. See -// RFC section 3.4 for details on the format. -// Returns a non-nil error if there was a problem. -func ReadTimestampedEntryInto(r io.Reader, t *TimestampedEntry) error { - var err error - if err = binary.Read(r, binary.BigEndian, &t.Timestamp); err != nil { - return err - } - if err = binary.Read(r, binary.BigEndian, &t.EntryType); err != nil { - return err - } - switch t.EntryType { - case X509LogEntryType: - if t.X509Entry, err = readVarBytes(r, CertificateLengthBytes); err != nil { - return err - } - case PrecertLogEntryType: - if err := binary.Read(r, binary.BigEndian, &t.PrecertEntry.IssuerKeyHash); err != nil { - return err - } - if t.PrecertEntry.TBSCertificate, err = readVarBytes(r, PreCertificateLengthBytes); err != nil { - return err - } - default: - return fmt.Errorf("unknown EntryType: %d", t.EntryType) - } - t.Extensions, err = readVarBytes(r, ExtensionsLengthBytes) - return nil -} - -// ReadMerkleTreeLeaf parses the byte-stream representation of a MerkleTreeLeaf -// and returns a pointer to a new MerkleTreeLeaf structure containing the -// parsed data. -// See RFC section 3.4 for details on the format. -// Returns a pointer to a new MerkleTreeLeaf or non-nil error if there was a -// problem -func ReadMerkleTreeLeaf(r io.Reader) (*MerkleTreeLeaf, error) { - var m MerkleTreeLeaf - if err := binary.Read(r, binary.BigEndian, &m.Version); err != nil { - return nil, err - } - if m.Version != V1 { - return nil, fmt.Errorf("unknown Version %d", m.Version) - } - if err := binary.Read(r, binary.BigEndian, &m.LeafType); err != nil { - return nil, err - } - if m.LeafType != TimestampedEntryLeafType { - return nil, fmt.Errorf("unknown LeafType %d", m.LeafType) - } - if err := ReadTimestampedEntryInto(r, &m.TimestampedEntry); err != nil { - return nil, err - } - return &m, nil -} - -// UnmarshalX509ChainArray unmarshalls the contents of the "chain:" entry in a -// GetEntries response in the case where the entry refers to an X509 leaf. -func UnmarshalX509ChainArray(b []byte) ([]ASN1Cert, error) { - return readASN1CertList(bytes.NewReader(b), CertificateChainLengthBytes, CertificateLengthBytes) -} - -// UnmarshalPrecertChainArray unmarshalls the contents of the "chain:" entry in -// a GetEntries response in the case where the entry refers to a Precertificate -// leaf. -func UnmarshalPrecertChainArray(b []byte) ([]ASN1Cert, error) { - var chain []ASN1Cert - - reader := bytes.NewReader(b) - // read the pre-cert entry: - precert, err := readVarBytes(reader, CertificateLengthBytes) - if err != nil { - return chain, err - } - chain = append(chain, precert) - // and then read and return the chain up to the root: - remainingChain, err := readASN1CertList(reader, CertificateChainLengthBytes, CertificateLengthBytes) - if err != nil { - return chain, err - } - chain = append(chain, remainingChain...) - return chain, nil -} - -// UnmarshalDigitallySigned reconstructs a DigitallySigned structure from a Reader -func UnmarshalDigitallySigned(r io.Reader) (*DigitallySigned, error) { - var h byte - if err := binary.Read(r, binary.BigEndian, &h); err != nil { - return nil, fmt.Errorf("failed to read HashAlgorithm: %v", err) - } - - var s byte - if err := binary.Read(r, binary.BigEndian, &s); err != nil { - return nil, fmt.Errorf("failed to read SignatureAlgorithm: %v", err) - } - - sig, err := readVarBytes(r, SignatureLengthBytes) - if err != nil { - return nil, fmt.Errorf("failed to read Signature bytes: %v", err) - } - - return &DigitallySigned{ - HashAlgorithm: HashAlgorithm(h), - SignatureAlgorithm: SignatureAlgorithm(s), - Signature: sig, - }, nil -} - -// MarshalDigitallySigned marshalls a DigitallySigned structure into a byte array -func MarshalDigitallySigned(ds DigitallySigned) ([]byte, error) { - var b bytes.Buffer - if err := b.WriteByte(byte(ds.HashAlgorithm)); err != nil { - return nil, fmt.Errorf("failed to write HashAlgorithm: %v", err) - } - if err := b.WriteByte(byte(ds.SignatureAlgorithm)); err != nil { - return nil, fmt.Errorf("failed to write SignatureAlgorithm: %v", err) - } - if err := writeVarBytes(&b, ds.Signature, SignatureLengthBytes); err != nil { - return nil, fmt.Errorf("failed to write HashAlgorithm: %v", err) - } - return b.Bytes(), nil -} - -func checkCertificateFormat(cert ASN1Cert) error { - if len(cert) == 0 { - return errors.New("certificate is zero length") - } - if len(cert) > MaxCertificateLength { - return errors.New("certificate too large") - } - return nil -} - -func checkExtensionsFormat(ext CTExtensions) error { - if len(ext) > MaxExtensionsLength { - return errors.New("extensions too large") - } - return nil -} - -func serializeV1CertSCTSignatureInput(timestamp uint64, cert ASN1Cert, ext CTExtensions) ([]byte, error) { - if err := checkCertificateFormat(cert); err != nil { - return nil, err - } - if err := checkExtensionsFormat(ext); err != nil { - return nil, err - } - var buf bytes.Buffer - if err := binary.Write(&buf, binary.BigEndian, V1); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, CertificateTimestampSignatureType); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, timestamp); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, X509LogEntryType); err != nil { - return nil, err - } - if err := writeVarBytes(&buf, cert, CertificateLengthBytes); err != nil { - return nil, err - } - if err := writeVarBytes(&buf, ext, ExtensionsLengthBytes); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func serializeV1PrecertSCTSignatureInput(timestamp uint64, issuerKeyHash [issuerKeyHashLength]byte, tbs []byte, ext CTExtensions) ([]byte, error) { - if err := checkCertificateFormat(tbs); err != nil { - return nil, err - } - if err := checkExtensionsFormat(ext); err != nil { - return nil, err - } - var buf bytes.Buffer - if err := binary.Write(&buf, binary.BigEndian, V1); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, CertificateTimestampSignatureType); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, timestamp); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, PrecertLogEntryType); err != nil { - return nil, err - } - if _, err := buf.Write(issuerKeyHash[:]); err != nil { - return nil, err - } - if err := writeVarBytes(&buf, tbs, CertificateLengthBytes); err != nil { - return nil, err - } - if err := writeVarBytes(&buf, ext, ExtensionsLengthBytes); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func serializeV1SCTSignatureInput(sct SignedCertificateTimestamp, entry LogEntry) ([]byte, error) { - if sct.SCTVersion != V1 { - return nil, fmt.Errorf("unsupported SCT version, expected V1, but got %s", sct.SCTVersion) - } - if entry.Leaf.LeafType != TimestampedEntryLeafType { - return nil, fmt.Errorf("Unsupported leaf type %s", entry.Leaf.LeafType) - } - switch entry.Leaf.TimestampedEntry.EntryType { - case X509LogEntryType: - return serializeV1CertSCTSignatureInput(sct.Timestamp, entry.Leaf.TimestampedEntry.X509Entry, entry.Leaf.TimestampedEntry.Extensions) - case PrecertLogEntryType: - return serializeV1PrecertSCTSignatureInput(sct.Timestamp, entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash, - entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate, - entry.Leaf.TimestampedEntry.Extensions) - default: - return nil, fmt.Errorf("unknown TimestampedEntryLeafType %s", entry.Leaf.TimestampedEntry.EntryType) - } -} - -// SerializeSCTSignatureInput serializes the passed in sct and log entry into -// the correct format for signing. -func SerializeSCTSignatureInput(sct SignedCertificateTimestamp, entry LogEntry) ([]byte, error) { - switch sct.SCTVersion { - case V1: - return serializeV1SCTSignatureInput(sct, entry) - default: - return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion) - } -} - -func serializeV1SCT(sct SignedCertificateTimestamp) ([]byte, error) { - if err := checkExtensionsFormat(sct.Extensions); err != nil { - return nil, err - } - var buf bytes.Buffer - if err := binary.Write(&buf, binary.BigEndian, V1); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, sct.LogID); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, sct.Timestamp); err != nil { - return nil, err - } - if err := writeVarBytes(&buf, sct.Extensions, ExtensionsLengthBytes); err != nil { - return nil, err - } - sig, err := MarshalDigitallySigned(sct.Signature) - if err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, sig); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// SerializeSCT serializes the passed in sct into the format specified -// by RFC6962 section 3.2 -func SerializeSCT(sct SignedCertificateTimestamp) ([]byte, error) { - switch sct.SCTVersion { - case V1: - return serializeV1SCT(sct) - default: - return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion) - } -} - -func deserializeSCTV1(r io.Reader, sct *SignedCertificateTimestamp) error { - if err := binary.Read(r, binary.BigEndian, &sct.LogID); err != nil { - return err - } - if err := binary.Read(r, binary.BigEndian, &sct.Timestamp); err != nil { - return err - } - ext, err := readVarBytes(r, ExtensionsLengthBytes) - if err != nil { - return err - } - sct.Extensions = ext - ds, err := UnmarshalDigitallySigned(r) - if err != nil { - return err - } - sct.Signature = *ds - return nil -} - -func DeserializeSCT(r io.Reader) (*SignedCertificateTimestamp, error) { - var sct SignedCertificateTimestamp - if err := binary.Read(r, binary.BigEndian, &sct.SCTVersion); err != nil { - return nil, err - } - switch sct.SCTVersion { - case V1: - return &sct, deserializeSCTV1(r, &sct) - default: - return nil, fmt.Errorf("unknown SCT version %d", sct.SCTVersion) - } -} - -func serializeV1STHSignatureInput(sth SignedTreeHead) ([]byte, error) { - if sth.Version != V1 { - return nil, fmt.Errorf("invalid STH version %d", sth.Version) - } - if sth.TreeSize < 0 { - return nil, fmt.Errorf("invalid tree size %d", sth.TreeSize) - } - if len(sth.SHA256RootHash) != crypto.SHA256.Size() { - return nil, fmt.Errorf("invalid TreeHash length, got %d expected %d", len(sth.SHA256RootHash), crypto.SHA256.Size()) - } - - var buf bytes.Buffer - if err := binary.Write(&buf, binary.BigEndian, V1); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, TreeHashSignatureType); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, sth.Timestamp); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, sth.TreeSize); err != nil { - return nil, err - } - if err := binary.Write(&buf, binary.BigEndian, sth.SHA256RootHash); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// SerializeSTHSignatureInput serializes the passed in sth into the correct -// format for signing. -func SerializeSTHSignatureInput(sth SignedTreeHead) ([]byte, error) { - switch sth.Version { - case V1: - return serializeV1STHSignatureInput(sth) - default: - return nil, fmt.Errorf("unsupported STH version %d", sth.Version) - } -} diff --git a/ct/signatures.go b/ct/signatures.go deleted file mode 100644 index 0921674..0000000 --- a/ct/signatures.go +++ /dev/null @@ -1,109 +0,0 @@ -package ct - -import ( - "crypto" - "crypto/ecdsa" - "crypto/rsa" - "crypto/sha256" - "crypto/x509" - "encoding/asn1" - "encoding/pem" - "errors" - "fmt" - "math/big" -) - -// PublicKeyFromPEM parses a PEM formatted block and returns the public key contained within and any remaining unread bytes, or an error. -func PublicKeyFromPEM(b []byte) (crypto.PublicKey, SHA256Hash, []byte, error) { - p, rest := pem.Decode(b) - if p == nil { - return nil, [sha256.Size]byte{}, rest, fmt.Errorf("no PEM block found in %s", string(b)) - } - k, err := x509.ParsePKIXPublicKey(p.Bytes) - return k, sha256.Sum256(p.Bytes), rest, err -} - -// SignatureVerifier can verify signatures on SCTs and STHs -type SignatureVerifier struct { - pubKey crypto.PublicKey -} - -// NewSignatureVerifier creates a new SignatureVerifier using the passed in PublicKey. -func NewSignatureVerifier(pk crypto.PublicKey) (*SignatureVerifier, error) { - switch pkType := pk.(type) { - case *rsa.PublicKey: - case *ecdsa.PublicKey: - default: - return nil, fmt.Errorf("Unsupported public key type %v", pkType) - } - - return &SignatureVerifier{ - pubKey: pk, - }, nil -} - -// verifySignature verifies that the passed in signature over data was created by our PublicKey. -// Currently, only SHA256 is supported as a HashAlgorithm, and only ECDSA and RSA signatures are supported. -func (s SignatureVerifier) verifySignature(data []byte, sig DigitallySigned) error { - if sig.HashAlgorithm != SHA256 { - return fmt.Errorf("unsupported HashAlgorithm in signature: %v", sig.HashAlgorithm) - } - - hasherType := crypto.SHA256 - hasher := hasherType.New() - if _, err := hasher.Write(data); err != nil { - return fmt.Errorf("failed to write to hasher: %v", err) - } - hash := hasher.Sum([]byte{}) - - switch sig.SignatureAlgorithm { - case RSA: - rsaKey, ok := s.pubKey.(*rsa.PublicKey) - if !ok { - return fmt.Errorf("cannot verify RSA signature with %T key", s.pubKey) - } - if err := rsa.VerifyPKCS1v15(rsaKey, hasherType, hash, sig.Signature); err != nil { - return fmt.Errorf("failed to verify rsa signature: %v", err) - } - case ECDSA: - ecdsaKey, ok := s.pubKey.(*ecdsa.PublicKey) - if !ok { - return fmt.Errorf("cannot verify ECDSA signature with %T key", s.pubKey) - } - var ecdsaSig struct { - R, S *big.Int - } - rest, err := asn1.Unmarshal(sig.Signature, &ecdsaSig) - if err != nil { - return fmt.Errorf("failed to unmarshal ECDSA signature: %v", err) - } - if len(rest) != 0 { - return fmt.Errorf("Garbage following signature %v", rest) - } - - if !ecdsa.Verify(ecdsaKey, hash, ecdsaSig.R, ecdsaSig.S) { - return errors.New("failed to verify ecdsa signature") - } - default: - return fmt.Errorf("unsupported signature type %v", sig.SignatureAlgorithm) - } - return nil -} - -// VerifySCTSignature verifies that the SCT's signature is valid for the given LogEntry -func (s SignatureVerifier) VerifySCTSignature(sct SignedCertificateTimestamp, entry LogEntry) error { - sctData, err := SerializeSCTSignatureInput(sct, entry) - if err != nil { - return err - } - return s.verifySignature(sctData, sct.Signature) -} - -// VerifySTHSignature verifies that the STH's signature is valid. -func (s SignatureVerifier) VerifySTHSignature(sth SignedTreeHead) error { - sthData, err := SerializeSTHSignatureInput(sth) - if err != nil { - return err - } - return s.verifySignature(sthData, sth.TreeHeadSignature) -} diff --git a/ct/types.go b/ct/types.go deleted file mode 100644 index a79f405..0000000 --- a/ct/types.go +++ /dev/null @@ -1,333 +0,0 @@ -package ct - -import ( - "bytes" - "crypto/sha256" - "encoding/base64" - "encoding/json" - "fmt" - "time" -) - -const ( - issuerKeyHashLength = 32 -) - -/////////////////////////////////////////////////////////////////////////////// -// The following structures represent those outlined in the RFC6962 document: -/////////////////////////////////////////////////////////////////////////////// - -// LogEntryType represents the LogEntryType enum from section 3.1 of the RFC: -// enum { x509_entry(0), precert_entry(1), (65535) } LogEntryType; -type LogEntryType uint16 - -func (e LogEntryType) String() string { - switch e { - case X509LogEntryType: - return "X509LogEntryType" - case PrecertLogEntryType: - return "PrecertLogEntryType" - } - panic(fmt.Sprintf("No string defined for LogEntryType constant value %d", e)) -} - -// LogEntryType constants, see section 3.1 of RFC6962. -const ( - X509LogEntryType LogEntryType = 0 - PrecertLogEntryType LogEntryType = 1 -) - -// MerkleLeafType represents the MerkleLeafType enum from section 3.4 of the -// RFC: enum { timestamped_entry(0), (255) } MerkleLeafType; -type MerkleLeafType uint8 - -func (m MerkleLeafType) String() string { - switch m { - case TimestampedEntryLeafType: - return "TimestampedEntryLeafType" - default: - return fmt.Sprintf("UnknownLeafType(%d)", m) - } -} - -// MerkleLeafType constants, see section 3.4 of the RFC. -const ( - TimestampedEntryLeafType MerkleLeafType = 0 // Entry type for an SCT -) - -// Version represents the Version enum from section 3.2 of the RFC: -// enum { v1(0), (255) } Version; -type Version uint8 - -func (v Version) String() string { - switch v { - case V1: - return "V1" - default: - return fmt.Sprintf("UnknownVersion(%d)", v) - } -} - -// CT Version constants, see section 3.2 of the RFC. -const ( - V1 Version = 0 -) - -// SignatureType differentiates STH signatures from SCT signatures, see RFC -// section 3.2 -type SignatureType uint8 - -func (st SignatureType) String() string { - switch st { - case CertificateTimestampSignatureType: - return "CertificateTimestamp" - case TreeHashSignatureType: - return "TreeHash" - default: - return fmt.Sprintf("UnknownSignatureType(%d)", st) - } -} - -// SignatureType constants, see RFC section 3.2 -const ( - CertificateTimestampSignatureType SignatureType = 0 - TreeHashSignatureType SignatureType = 1 -) - -// ASN1Cert type for holding the raw DER bytes of an ASN.1 Certificate -// (section 3.1) -type ASN1Cert []byte - -// PreCert represents a Precertificate (section 3.2) -type PreCert struct { - IssuerKeyHash [issuerKeyHashLength]byte - TBSCertificate []byte -} - -// CTExtensions is a representation of the raw bytes of any CtExtension -// structure (see section 3.2) -type CTExtensions []byte - -// MerkleTreeNode represents an internal node in the CT tree -type MerkleTreeNode []byte - -// ConsistencyProof represents a CT consistency proof (see sections 2.1.2 and -// 4.4) -type ConsistencyProof []MerkleTreeNode - -// AuditPath represents a CT inclusion proof (see sections 2.1.1 and 4.5) -type AuditPath []MerkleTreeNode - -// LeafInput represents a serialized MerkleTreeLeaf structure -type LeafInput []byte - -// HashAlgorithm from the DigitallySigned struct -type HashAlgorithm byte - -// HashAlgorithm constants -const ( - None HashAlgorithm = 0 - MD5 HashAlgorithm = 1 - SHA1 HashAlgorithm = 2 - SHA224 HashAlgorithm = 3 - SHA256 HashAlgorithm = 4 - SHA384 HashAlgorithm = 5 - SHA512 HashAlgorithm = 6 -) - -func (h HashAlgorithm) String() string { - switch h { - case None: - return "None" - case MD5: - return "MD5" - case SHA1: - return "SHA1" - case SHA224: - return "SHA224" - case SHA256: - return "SHA256" - case SHA384: - return "SHA384" - case SHA512: - return "SHA512" - default: - return fmt.Sprintf("UNKNOWN(%d)", h) - } -} - -// SignatureAlgorithm from the DigitallySigned struct -type SignatureAlgorithm byte - -// SignatureAlgorithm constants -const ( - Anonymous SignatureAlgorithm = 0 - RSA SignatureAlgorithm = 1 - DSA SignatureAlgorithm = 2 - ECDSA SignatureAlgorithm = 3 -) - -func (s SignatureAlgorithm) String() string { - switch s { - case Anonymous: - return "Anonymous" - case RSA: - return "RSA" - case DSA: - return "DSA" - case ECDSA: - return "ECDSA" - default: - return fmt.Sprintf("UNKNOWN(%d)", s) - } -} - -// DigitallySigned represents an RFC5246 DigitallySigned structure -type DigitallySigned struct { - HashAlgorithm HashAlgorithm - SignatureAlgorithm SignatureAlgorithm - Signature []byte -} - -// FromBase64String populates the DigitallySigned structure from the base64 data passed in. -// Returns an error if the base64 data is invalid. -func (d *DigitallySigned) FromBase64String(b64 string) error { - raw, err := base64.StdEncoding.DecodeString(b64) - if err != nil { - return fmt.Errorf("failed to unbase64 DigitallySigned: %v", err) - } - ds, err := UnmarshalDigitallySigned(bytes.NewReader(raw)) - if err != nil { - return fmt.Errorf("failed to unmarshal DigitallySigned: %v", err) - } - *d = *ds - return nil -} - -// Base64String returns the base64 representation of the DigitallySigned struct. -func (d DigitallySigned) Base64String() (string, error) { - b, err := MarshalDigitallySigned(d) - if err != nil { - return "", err - } - return base64.StdEncoding.EncodeToString(b), nil -} - -// MarshalJSON implements the json.Marshaller interface. -func (d DigitallySigned) MarshalJSON() ([]byte, error) { - b64, err := d.Base64String() - if err != nil { - return []byte{}, err - } - return []byte(`"` + b64 + `"`), nil -} - -// UnmarshalJSON implements the json.Unmarshaler interface. -func (d *DigitallySigned) UnmarshalJSON(b []byte) error { - var content string - if err := json.Unmarshal(b, &content); err != nil { - return fmt.Errorf("failed to unmarshal DigitallySigned: %v", err) - } - return d.FromBase64String(content) -} - -// LogEntry represents the contents of an entry in a CT log, see section 3.1. -type LogEntry struct { - Index int64 - Leaf MerkleTreeLeaf - Chain []ASN1Cert - LeafBytes []byte -} - -// SHA256Hash represents the output from the SHA256 hash function. -type SHA256Hash [sha256.Size]byte - -// FromBase64String populates the SHA256 struct with the contents of the base64 data passed in. -func (s *SHA256Hash) FromBase64String(b64 string) error { - bs, err := base64.StdEncoding.DecodeString(b64) - if err != nil { - return fmt.Errorf("failed to unbase64 LogID: %v", err) - } - if len(bs) != sha256.Size { - return fmt.Errorf("invalid SHA256 length, expected 32 but got %d", len(bs)) - } - copy(s[:], bs) - return nil -} - -// Base64String returns the base64 representation of this SHA256Hash. -func (s SHA256Hash) Base64String() string { - return base64.StdEncoding.EncodeToString(s[:]) -} - -// Returns the raw base64url representation of this SHA256Hash. -func (s SHA256Hash) Base64URLString() string { - return base64.RawURLEncoding.EncodeToString(s[:]) -} - -// MarshalJSON implements the json.Marshaller interface for SHA256Hash. -func (s SHA256Hash) MarshalJSON() ([]byte, error) { - return []byte(`"` + s.Base64String() + `"`), nil -} - -// UnmarshalJSON implements the json.Unmarshaller interface. -func (s *SHA256Hash) UnmarshalJSON(b []byte) error { - var content string - if err := json.Unmarshal(b, &content); err != nil { - return fmt.Errorf("failed to unmarshal SHA256Hash: %v", err) - } - return s.FromBase64String(content) -} - -// SignedTreeHead represents the structure returned by the get-sth CT method -// after base64 decoding. See sections 3.5 and 4.3 in the RFC) -type SignedTreeHead struct { - Version Version `json:"sth_version"` // The version of the protocol to which the STH conforms - TreeSize uint64 `json:"tree_size"` // The number of entries in the new tree - Timestamp uint64 `json:"timestamp"` // The time at which the STH was created - SHA256RootHash SHA256Hash `json:"sha256_root_hash"` // The root hash of the log's Merkle tree - TreeHeadSignature DigitallySigned `json:"tree_head_signature"` // The Log's signature for this STH (see RFC section 3.5) - LogID SHA256Hash `json:"log_id"` // The SHA256 hash of the log's public key -} - -func (sth *SignedTreeHead) TimestampTime() time.Time { - return time.Unix(int64(sth.Timestamp/1000), int64(sth.Timestamp%1000)*1_000_000).UTC() -} - -// SignedCertificateTimestamp represents the structure returned by the -// add-chain and add-pre-chain methods after base64 decoding. (see RFC sections -// 3.2 ,4.1 and 4.2) -type SignedCertificateTimestamp struct { - SCTVersion Version `json:"sct_version"` // The version of the protocol to which the SCT conforms - LogID SHA256Hash `json:"id"` // the SHA-256 hash of the log's public key, calculated over - // the DER encoding of the key represented as SubjectPublicKeyInfo. - Timestamp uint64 `json:"timestamp"` // Timestamp (in ms since unix epoch) at which the SCT was issued - Extensions CTExtensions `json:"extensions"` // For future extensions to the protocol - Signature DigitallySigned `json:"signature"` // The Log's signature for this SCT -} - -func (s SignedCertificateTimestamp) String() string { - return fmt.Sprintf("{Version:%d LogId:%s Timestamp:%d Extensions:'%s' Signature:%v}", s.SCTVersion, - base64.StdEncoding.EncodeToString(s.LogID[:]), - s.Timestamp, - s.Extensions, - s.Signature) -} - -// TimestampedEntry is part of the MerkleTreeLeaf structure. -// See RFC section 3.4 -type TimestampedEntry struct { - Timestamp uint64 - EntryType LogEntryType - X509Entry ASN1Cert - PrecertEntry PreCert - Extensions CTExtensions -} - -// MerkleTreeLeaf represents the deserialized structure of the hash input for the -// leaves of a log's Merkle tree. See RFC section 3.4 -type MerkleTreeLeaf struct { - Version Version // the version of the protocol to which the MerkleTreeLeaf corresponds - LeafType MerkleLeafType // The type of the leaf input, currently only TimestampedEntry can exist - TimestampedEntry TimestampedEntry // The entry data itself -} diff --git a/helpers.go b/helpers.go index cd524bf..56c75f7 100644 --- a/helpers.go +++ b/helpers.go @@ -10,16 +10,9 @@ package certspotter import ( - "fmt" "math/big" - - "software.sslmate.com/src/certspotter/ct" ) -func IsPrecert(entry *ct.LogEntry) bool { - return entry.Leaf.TimestampedEntry.EntryType == ct.PrecertLogEntryType -} - type CertInfo struct { TBS *TBSCertificate @@ -68,19 +61,6 @@ func MakeCertInfoFromRawCert(certBytes []byte) (*CertInfo, error) { return MakeCertInfoFromRawTBS(cert.GetRawTBSCertificate()) } -func MakeCertInfoFromLogEntry(entry *ct.LogEntry) (*CertInfo, error) { - switch entry.Leaf.TimestampedEntry.EntryType { - case ct.X509LogEntryType: - return MakeCertInfoFromRawCert(entry.Leaf.TimestampedEntry.X509Entry) - - case ct.PrecertLogEntryType: - return MakeCertInfoFromRawTBS(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate) - - default: - return nil, fmt.Errorf("MakeCertInfoFromCTEntry: unknown CT entry type (neither X509 nor precert)") - } -} - func MatchesWildcard(dnsName string, pattern string) bool { for len(pattern) > 0 { if pattern[0] == '*' { diff --git a/loglist/schema.go b/loglist/schema.go index 7d070aa..bb210d8 100644 --- a/loglist/schema.go +++ b/loglist/schema.go @@ -12,7 +12,7 @@ package loglist import ( "time" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" ) type List struct { @@ -30,7 +30,7 @@ type Operator struct { type Log struct { Key []byte `json:"key"` - LogID ct.SHA256Hash `json:"log_id"` + LogID cttypes.LogID `json:"log_id"` MMD int `json:"mmd"` URL string `json:"url,omitempty"` // only for rfc6962 logs SubmissionURL string `json:"submission_url,omitempty"` // only for static-ct-api logs @@ -44,6 +44,9 @@ type Log struct { EndExclusive time.Time `json:"end_exclusive"` } `json:"temporal_interval"` + DownloadWorkers int `json:"certspotter_download_workers,omitempty"` + DownloadJobSize int `json:"certspotter_download_job_size,omitempty"` + // TODO: add previous_operators } diff --git a/man/certspotter-script.md b/man/certspotter-script.md index 65441d4..47b1285 100644 --- a/man/certspotter-script.md +++ b/man/certspotter-script.md @@ -123,6 +123,10 @@ The following environment variables are set for `discovered_cert` events: : Error parsing the serial number, if any. If this variable is set, then `SERIAL` is unset. +`CHAIN_ERROR` + +: Error building or verifying the certificate chain, if any. If this variable is set, then the certificate chain in `CERT_FILENAME` may be incomplete or invalid. + ## Malformed certificate information The following environment variables are set for `malformed_cert` events: diff --git a/monitor/daemon.go b/monitor/daemon.go index 79ca9ff..ddf20fd 100644 --- a/monitor/daemon.go +++ b/monitor/daemon.go @@ -63,7 +63,7 @@ func (daemon *daemon) healthCheck(ctx context.Context) error { for _, task := range daemon.tasks { if err := healthCheckLog(ctx, daemon.config, task.log); err != nil { - return fmt.Errorf("error checking health of log %q: %w", task.log.URL, err) + return fmt.Errorf("error checking health of log %q: %w", task.log.GetMonitoringURL(), err) } } return nil @@ -75,12 +75,12 @@ func (daemon *daemon) startTask(ctx context.Context, ctlog *loglist.Log) task { defer cancel() err := monitorLogContinously(ctx, daemon.config, ctlog) if daemon.config.Verbose { - log.Printf("task for log %s stopped with error %s", ctlog.URL, err) + log.Printf("task for log %s stopped with error %s", ctlog.GetMonitoringURL(), err) } if ctx.Err() == context.Canceled && errors.Is(err, context.Canceled) { return nil } else { - return fmt.Errorf("error while monitoring %s: %w", ctlog.URL, err) + return fmt.Errorf("error while monitoring %s: %w", ctlog.GetMonitoringURL(), err) } }) return task{log: ctlog, stop: cancel} @@ -113,7 +113,7 @@ func (daemon *daemon) loadLogList(ctx context.Context) error { continue } if daemon.config.Verbose { - log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.URL) + log.Printf("starting task for log %s (%s)", logID.Base64String(), ctlog.GetMonitoringURL()) } daemon.tasks[logID] = daemon.startTask(ctx, ctlog) } diff --git a/monitor/discoveredcert.go b/monitor/discoveredcert.go index 46fc7ba..a5b4055 100644 --- a/monitor/discoveredcert.go +++ b/monitor/discoveredcert.go @@ -18,17 +18,18 @@ import ( "time" "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" ) type DiscoveredCert struct { WatchItem WatchItem LogEntry *LogEntry Info *certspotter.CertInfo - Chain []ct.ASN1Cert // first entry is the leaf certificate or precertificate - TBSSHA256 [32]byte // computed over Info.TBS.Raw - SHA256 [32]byte // computed over Chain[0] - PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes + Chain []cttypes.ASN1Cert // first entry is the leaf certificate or precertificate + ChainError error // any error generating or validating Chain; if non-nil, Chain may be partial or incorrect + TBSSHA256 [32]byte // computed over Info.TBS.Raw + SHA256 [32]byte // computed over Chain[0] + PubkeySHA256 [32]byte // computed over Info.TBS.PublicKey.FullBytes Identifiers *certspotter.Identifiers } @@ -40,6 +41,9 @@ type certPaths struct { func (cert *DiscoveredCert) pemChain() []byte { var buffer bytes.Buffer + if cert.ChainError != nil { + fmt.Fprintln(&buffer, "Warning: this chain may be incomplete or invalid: %s", cert.ChainError) + } for _, certBytes := range cert.Chain { if err := pem.Encode(&buffer, &pem.Block{ Type: "CERTIFICATE", @@ -88,7 +92,7 @@ func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string { "EVENT=discovered_cert", "SUMMARY=" + certNotificationSummary(cert), "CERT_PARSEABLE=yes", // backwards compat with pre-0.15.0; not documented - "LOG_URI=" + cert.LogEntry.Log.URL, + "LOG_URI=" + cert.LogEntry.Log.GetMonitoringURL(), "ENTRY_INDEX=" + fmt.Sprint(cert.LogEntry.Index), "WATCH_ITEM=" + cert.WatchItem.String(), "TBS_SHA256=" + hex.EncodeToString(cert.TBSSHA256[:]), @@ -133,6 +137,10 @@ func certNotificationEnviron(cert *DiscoveredCert, paths *certPaths) []string { env = append(env, "SERIAL_PARSE_ERROR="+cert.Info.SerialNumberParseError.Error()) } + if cert.ChainError != nil { + env = append(env, "CHAIN_ERROR="+cert.ChainError.Error()) + } + return env } @@ -162,8 +170,11 @@ func certNotificationText(cert *DiscoveredCert, paths *certPaths) string { writeField("Not Before", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) writeField("Not After", fmt.Sprintf("[unable to parse: %s]", cert.Info.ValidityParseError)) } - writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.URL)) + writeField("Log Entry", fmt.Sprintf("%d @ %s", cert.LogEntry.Index, cert.LogEntry.Log.GetMonitoringURL())) writeField("crt.sh", "https://crt.sh/?sha256="+hex.EncodeToString(cert.SHA256[:])) + if cert.ChainError != nil { + writeField("Error Building Chain", cert.ChainError.Error()) + } if paths != nil { writeField("Filename", paths.certPath) } diff --git a/monitor/errors.go b/monitor/errors.go index c772121..ca12889 100644 --- a/monitor/errors.go +++ b/monitor/errors.go @@ -22,7 +22,7 @@ func recordError(ctx context.Context, config *Config, ctlog *loglist.Log, errToR if ctlog == nil { log.Print(errToRecord) } else { - log.Print(ctlog.URL, ": ", errToRecord) + log.Print(ctlog.GetMonitoringURL(), ": ", errToRecord) } } } diff --git a/monitor/fsstate.go b/monitor/fsstate.go index 70a2962..c1131a6 100644 --- a/monitor/fsstate.go +++ b/monitor/fsstate.go @@ -21,8 +21,9 @@ import ( "path/filepath" "strings" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" "software.sslmate.com/src/certspotter/loglist" + "software.sslmate.com/src/certspotter/merkletree" ) type FilesystemState struct { @@ -77,17 +78,17 @@ func (s *FilesystemState) StoreLogState(ctx context.Context, logID LogID, state return writeJSONFile(filePath, state, 0666) } -func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error { +func (s *FilesystemState) StoreSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error { sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") return storeSTHInDir(sthsDirPath, sth) } -func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*ct.SignedTreeHead, error) { +func (s *FilesystemState) LoadSTHs(ctx context.Context, logID LogID) ([]*cttypes.SignedTreeHead, error) { sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") return loadSTHsFromDir(sthsDirPath) } -func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *ct.SignedTreeHead) error { +func (s *FilesystemState) RemoveSTH(ctx context.Context, logID LogID, sth *cttypes.SignedTreeHead) error { sthsDirPath := filepath.Join(s.logStateDir(logID), "unverified_sths") return removeSTHFromDir(sthsDirPath, sth) } @@ -154,24 +155,17 @@ func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEn textPath = filepath.Join(dirPath, fmt.Sprintf("%d.txt", entry.Index)) ) - summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.URL) - - entryJSON := struct { - LeafInput []byte `json:"leaf_input"` - ExtraData []byte `json:"extra_data"` - }{ - LeafInput: entry.LeafInput, - ExtraData: entry.ExtraData, - } + summary := fmt.Sprintf("Unable to Parse Entry %d in %s", entry.Index, entry.Log.GetMonitoringURL()) + leafHash := merkletree.HashLeaf(entry.LeafInput()) text := new(strings.Builder) writeField := func(name string, value any) { fmt.Fprintf(text, "\t%13s = %s\n", name, value) } fmt.Fprintf(text, "Unable to determine if log entry matches your watchlist. Please file a bug report at https://github.com/SSLMate/certspotter/issues/new with the following details:\n") - writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.URL)) - writeField("Leaf Hash", entry.LeafHash.Base64String()) + writeField("Log Entry", fmt.Sprintf("%d @ %s", entry.Index, entry.Log.GetMonitoringURL())) + writeField("Leaf Hash", leafHash.Base64String()) writeField("Error", parseError.Error()) - if err := writeJSONFile(entryPath, entryJSON, 0666); err != nil { + if err := writeJSONFile(entryPath, entry.Entry, 0666); err != nil { return fmt.Errorf("error saving JSON file: %w", err) } if err := writeTextFile(textPath, text.String(), 0666); err != nil { @@ -181,9 +175,9 @@ func (s *FilesystemState) NotifyMalformedEntry(ctx context.Context, entry *LogEn environ := []string{ "EVENT=malformed_cert", "SUMMARY=" + summary, - "LOG_URI=" + entry.Log.URL, + "LOG_URI=" + entry.Log.GetMonitoringURL(), "ENTRY_INDEX=" + fmt.Sprint(entry.Index), - "LEAF_HASH=" + entry.LeafHash.Base64String(), + "LEAF_HASH=" + leafHash.Base64String(), "PARSE_ERROR=" + parseError.Error(), "ENTRY_FILENAME=" + entryPath, "TEXT_FILENAME=" + textPath, @@ -233,7 +227,7 @@ func (s *FilesystemState) NotifyError(ctx context.Context, ctlog *loglist.Log, e if ctlog == nil { log.Print(err) } else { - log.Print(ctlog.URL, ":", err) + log.Print(ctlog.GetMonitoringURL(), ":", err) } return nil } diff --git a/monitor/healthcheck.go b/monitor/healthcheck.go index 39811b6..17c939e 100644 --- a/monitor/healthcheck.go +++ b/monitor/healthcheck.go @@ -15,7 +15,7 @@ import ( "strings" "time" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" "software.sslmate.com/src/certspotter/loglist" ) @@ -31,7 +31,7 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err return nil } - if time.Since(state.LastSuccess) < config.HealthCheckInterval { + if state.VerifiedSTH != nil && time.Since(state.VerifiedSTH.TimestampTime()) < config.HealthCheckInterval { return nil } @@ -42,9 +42,8 @@ func healthCheckLog(ctx context.Context, config *Config, ctlog *loglist.Log) err if len(sths) == 0 { info := &StaleSTHInfo{ - Log: ctlog, - LastSuccess: state.LastSuccess, - LatestSTH: state.VerifiedSTH, + Log: ctlog, + LatestSTH: state.VerifiedSTH, } if err := config.State.NotifyHealthCheckFailure(ctx, ctlog, info); err != nil { return fmt.Errorf("error notifying about stale STH: %w", err) @@ -69,14 +68,13 @@ type HealthCheckFailure interface { } type StaleSTHInfo struct { - Log *loglist.Log - LastSuccess time.Time - LatestSTH *ct.SignedTreeHead // may be nil + Log *loglist.Log + LatestSTH *cttypes.SignedTreeHead // may be nil } type BacklogInfo struct { Log *loglist.Log - LatestSTH *ct.SignedTreeHead + LatestSTH *cttypes.SignedTreeHead Position uint64 } @@ -92,10 +90,10 @@ func (e *BacklogInfo) Backlog() uint64 { } func (e *StaleSTHInfo) Summary() string { - return fmt.Sprintf("Unable to contact %s since %s", e.Log.URL, e.LastSuccess) + return fmt.Sprintf("%s is out-of-date", e.Log.GetMonitoringURL()) } func (e *BacklogInfo) Summary() string { - return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.URL) + return fmt.Sprintf("Backlog of size %d from %s", e.Backlog(), e.Log.GetMonitoringURL()) } func (e *StaleLogListInfo) Summary() string { return fmt.Sprintf("Unable to retrieve log list since %s", e.LastSuccess) @@ -103,7 +101,7 @@ func (e *StaleLogListInfo) Summary() string { func (e *StaleSTHInfo) Text() string { text := new(strings.Builder) - fmt.Fprintf(text, "certspotter has been unable to contact %s since %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.URL, e.LastSuccess) + fmt.Fprintf(text, "certspotter has been unable to get up-to-date information about %s. Consequentially, certspotter may fail to notify you about certificates in this log.\n", e.Log.GetMonitoringURL()) fmt.Fprintf(text, "\n") fmt.Fprintf(text, "For details, see certspotter's stderr output.\n") fmt.Fprintf(text, "\n") @@ -116,7 +114,7 @@ func (e *StaleSTHInfo) Text() string { } func (e *BacklogInfo) Text() string { text := new(strings.Builder) - fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.URL) + fmt.Fprintf(text, "certspotter has been unable to download entries from %s in a timely manner. Consequentially, certspotter may be slow to notify you about certificates in this log.\n", e.Log.GetMonitoringURL()) fmt.Fprintf(text, "\n") fmt.Fprintf(text, "For more details, see certspotter's stderr output.\n") fmt.Fprintf(text, "\n") diff --git a/monitor/loglist.go b/monitor/loglist.go index cbbc502..3d88290 100644 --- a/monitor/loglist.go +++ b/monitor/loglist.go @@ -12,11 +12,11 @@ package monitor import ( "context" "fmt" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" "software.sslmate.com/src/certspotter/loglist" ) -type LogID = ct.SHA256Hash +type LogID = cttypes.LogID func getLogList(ctx context.Context, source string, token *loglist.ModificationToken) (map[LogID]*loglist.Log, *loglist.ModificationToken, error) { list, newToken, err := loglist.LoadIfModified(ctx, source, token) @@ -33,6 +33,13 @@ func getLogList(ctx context.Context, source string, token *loglist.ModificationT } logs[log.LogID] = log } + for logIndex := range list.Operators[operatorIndex].TiledLogs { + log := &list.Operators[operatorIndex].TiledLogs[logIndex] + if _, exists := logs[log.LogID]; exists { + return nil, nil, fmt.Errorf("log list contains more than one entry with ID %s", log.LogID.Base64String()) + } + logs[log.LogID] = log + } } return logs, newToken, nil } diff --git a/monitor/monitor.go b/monitor/monitor.go index 3877d1f..097519d 100644 --- a/monitor/monitor.go +++ b/monitor/monitor.go @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Opsmate, Inc. +// Copyright (C) 2025 Opsmate, Inc. // // This Source Code Form is subject to the terms of the Mozilla // Public License, v. 2.0. If a copy of the MPL was not distributed @@ -11,275 +11,503 @@ package monitor import ( "context" - "crypto/x509" "errors" "fmt" + "golang.org/x/sync/errgroup" "log" - "strings" + mathrand "math/rand/v2" + "net/url" + "slices" "time" - "software.sslmate.com/src/certspotter/ct" - "software.sslmate.com/src/certspotter/ct/client" + "software.sslmate.com/src/certspotter/ctclient" + "software.sslmate.com/src/certspotter/ctcrypto" + "software.sslmate.com/src/certspotter/cttypes" "software.sslmate.com/src/certspotter/loglist" "software.sslmate.com/src/certspotter/merkletree" + "software.sslmate.com/src/certspotter/sequencer" ) const ( - maxGetEntriesSize = 1000 - monitorLogInterval = 5 * time.Minute + getSTHInterval = 5 * time.Minute ) -func isFatalLogError(err error) bool { - return errors.Is(err, context.Canceled) +func downloadJobSize(ctlog *loglist.Log) uint64 { + if ctlog.IsStaticCTAPI() { + return ctclient.StaticTileWidth + } else if ctlog.DownloadJobSize != 0 { + return uint64(ctlog.DownloadJobSize) + } else { + return 256 + } } -func newLogClient(ctlog *loglist.Log) (*client.LogClient, error) { - logKey, err := x509.ParsePKIXPublicKey(ctlog.Key) - if err != nil { - return nil, fmt.Errorf("error parsing log key: %w", err) +func downloadWorkers(ctlog *loglist.Log) int { + if ctlog.DownloadWorkers != 0 { + return ctlog.DownloadWorkers + } else { + return 4 } - verifier, err := ct.NewSignatureVerifier(logKey) - if err != nil { - return nil, fmt.Errorf("error with log key: %w", err) - } - return client.NewWithVerifier(strings.TrimRight(ctlog.URL, "/"), verifier), nil } -func monitorLogContinously(ctx context.Context, config *Config, ctlog *loglist.Log) error { - logClient, err := newLogClient(ctlog) - if err != nil { - return err - } +type verifyEntriesError struct { + sth *cttypes.SignedTreeHead + entriesRootHash merkletree.Hash +} - ticker := time.NewTicker(monitorLogInterval) - defer ticker.Stop() +func (e *verifyEntriesError) Error() string { + return fmt.Sprintf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", e.sth.TreeSize, e.sth.RootHash, e.entriesRootHash) +} +func withRetry(ctx context.Context, maxRetries int, f func() error) error { + const minSleep = 1 * time.Second + const maxSleep = 10 * time.Minute + + numRetries := 0 for ctx.Err() == nil { - if err := monitorLog(ctx, config, ctlog, logClient); err != nil { + err := f() + if err == nil || errors.Is(err, context.Canceled) { return err } - select { - case <-ctx.Done(): - case <-ticker.C: + if maxRetries != -1 && numRetries >= maxRetries { + return fmt.Errorf("%w (retried %d times)", err, numRetries) } + upperBound := min(minSleep*(1< 0 && sths[0].TreeSize <= state.DownloadPosition.Size() { - // TODO-4: audit sths[0] against state.VerifiedSTH - if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil { - return fmt.Errorf("error removing STH: %w", err) - } - sths = sths[1:] - } - defer func() { if config.Verbose { - log.Printf("saving state in defer for %s", ctlog.URL) + log.Printf("saving state in defer for %s", ctlog.GetMonitoringURL()) } - if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil && returnedErr == nil { + storeCtx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + if err := config.State.StoreLogState(storeCtx, ctlog.LogID, state); err != nil && returnedErr == nil { returnedErr = fmt.Errorf("error storing log state: %w", err) } }() - if len(sths) == 0 { - state.LastSuccess = startTime.UTC() - return nil - } +retry: + position := state.DownloadPosition.Size() - var ( - downloadBegin = state.DownloadPosition.Size() - downloadEnd = sths[len(sths)-1].TreeSize - entries = make(chan client.GetEntriesItem, maxGetEntriesSize) - downloadErr error - ) - if config.Verbose { - log.Printf("downloading entries from %s in range [%d, %d)", ctlog.URL, downloadBegin, downloadEnd) + // generateBatchesWorker ==> downloadWorker ==> processWorker ==> saveStateWorker + + batches := make(chan *batch, downloadWorkers(ctlog)) + processedBatches := sequencer.New[batch](0, uint64(downloadWorkers(ctlog))*10) + + group, gctx := errgroup.WithContext(ctx) + group.Go(func() error { return getSTHWorker(gctx, config, ctlog, client) }) + group.Go(func() error { return generateBatchesWorker(gctx, config, ctlog, position, batches) }) + for range downloadWorkers(ctlog) { + downloadedBatches := make(chan *batch, 1) + group.Go(func() error { return downloadWorker(gctx, config, ctlog, client, batches, downloadedBatches) }) + group.Go(func() error { + return processWorker(gctx, config, ctlog, issuerGetter, downloadedBatches, processedBatches) + }) } - go func() { - defer close(entries) - downloadErr = downloadEntries(ctx, logClient, entries, downloadBegin, downloadEnd) - }() - for rawEntry := range entries { - entry := &LogEntry{ - Log: ctlog, - Index: state.DownloadPosition.Size(), - LeafInput: rawEntry.LeafInput, - ExtraData: rawEntry.ExtraData, - LeafHash: merkletree.HashLeaf(rawEntry.LeafInput), + group.Go(func() error { return saveStateWorker(gctx, config, ctlog, state, processedBatches) }) + + err = group.Wait() + if verifyErr := (*verifyEntriesError)(nil); errors.As(err, &verifyErr) { + recordError(ctx, config, ctlog, verifyErr) + state.rewindDownloadPosition() + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) } - if err := processLogEntry(ctx, config, entry); err != nil { - return fmt.Errorf("error processing entry %d: %w", entry.Index, err) - } - - state.DownloadPosition.Add(entry.LeafHash) - rootHash := state.DownloadPosition.CalculateRoot() - shouldSaveState := state.DownloadPosition.Size()%10000 == 0 - - for len(sths) > 0 && state.DownloadPosition.Size() == sths[0].TreeSize { - if merkletree.Hash(sths[0].SHA256RootHash) != rootHash { - recordError(ctx, config, ctlog, fmt.Errorf("error verifying at tree size %d: the STH root hash (%x) does not match the entries returned by the log (%x)", sths[0].TreeSize, sths[0].SHA256RootHash, rootHash)) - - state.DownloadPosition = state.VerifiedPosition - if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { - return fmt.Errorf("error storing log state: %w", err) - } - return nil - } - - state.VerifiedPosition = state.DownloadPosition - state.VerifiedSTH = sths[0] - shouldSaveState = true - if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil { - return fmt.Errorf("error removing verified STH: %w", err) - } - - sths = sths[1:] - } - - if shouldSaveState { - if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { - return fmt.Errorf("error storing state file: %w", err) - } + if err := sleep(ctx, 5*time.Minute); err != nil { + return err } + goto retry } - - if isFatalLogError(downloadErr) { - return downloadErr - } else if downloadErr != nil { - recordError(ctx, config, ctlog, fmt.Errorf("error downloading entries: %w", downloadErr)) - return nil - } - - if config.Verbose { - log.Printf("finished downloading entries from %s", ctlog.URL) - } - - state.LastSuccess = startTime.UTC() - return nil + return err } -func downloadEntries(ctx context.Context, logClient *client.LogClient, entriesChan chan<- client.GetEntriesItem, begin, end uint64) error { - for begin < end && ctx.Err() == nil { - size := end - begin - if size > maxGetEntriesSize { - size = maxGetEntriesSize - } - entries, err := logClient.GetRawEntries(ctx, begin, begin+size-1) +func getSTHWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log) error { + for ctx.Err() == nil { + sth, _, err := client.GetSTH(ctx) if err != nil { return err } - for _, entry := range entries { - if ctx.Err() != nil { - return ctx.Err() - } - select { - case <-ctx.Done(): - return ctx.Err() - case entriesChan <- entry: - } + if err := config.State.StoreSTH(ctx, ctlog.LogID, sth); err != nil { + return fmt.Errorf("error storing STH: %w", err) + } + if err := sleep(ctx, getSTHInterval); err != nil { + return err } - begin += uint64(len(entries)) } return ctx.Err() } -func reconstructTree(ctx context.Context, logClient *client.LogClient, sth *ct.SignedTreeHead) (*merkletree.CollapsedTree, error) { - if sth.TreeSize == 0 { - return merkletree.EmptyCollapsedTree(), nil - } - entries, err := logClient.GetRawEntries(ctx, sth.TreeSize-1, sth.TreeSize-1) - if err != nil { - return nil, err - } - leafHash := merkletree.HashLeaf(entries[0].LeafInput) - - var tree *merkletree.CollapsedTree - if sth.TreeSize > 1 { - // XXX: if leafHash is in the tree in more than one place, this might not return the proof that we need ... get-entry-and-proof avoids this problem but not all logs support it - auditPath, _, err := logClient.GetAuditProof(ctx, leafHash[:], sth.TreeSize) - if err != nil { - return nil, err - } - hashes := make([]merkletree.Hash, len(auditPath)) - for i := range hashes { - copy(hashes[i][:], auditPath[len(auditPath)-i-1]) - } - tree, err = merkletree.NewCollapsedTree(hashes, sth.TreeSize-1) - if err != nil { - return nil, fmt.Errorf("log returned invalid audit proof for %x to %d: %w", leafHash, sth.TreeSize, err) - } - } else { - tree = merkletree.EmptyCollapsedTree() - } - - tree.Add(leafHash) - rootHash := tree.CalculateRoot() - if rootHash != merkletree.Hash(sth.SHA256RootHash) { - return nil, fmt.Errorf("calculated root hash (%x) does not match signed tree head (%x) at size %d", rootHash, sth.SHA256RootHash, sth.TreeSize) - } - - return tree, nil +type batch struct { + number uint64 + begin, end uint64 + sths []*cttypes.SignedTreeHead // STHs with sizes in range [begin,end], sorted by TreeSize + entries []ctclient.Entry // in range [begin,end) +} + +func generateBatchesWorker(ctx context.Context, config *Config, ctlog *loglist.Log, position uint64, batches chan<- *batch) error { + ticker := time.NewTicker(15 * time.Second) + var number uint64 + for ctx.Err() == nil { + sths, err := config.State.LoadSTHs(ctx, ctlog.LogID) + if err != nil { + return fmt.Errorf("error loading STHs: %w", err) + } + for len(sths) > 0 && sths[0].TreeSize < position { + // TODO-4: audit sths[0] against log's verified STH + if err := config.State.RemoveSTH(ctx, ctlog.LogID, sths[0]); err != nil { + return fmt.Errorf("error removing STH: %w", err) + } + sths = sths[1:] + } + position, number, err = generateBatches(ctx, ctlog, position, number, sths, batches) + if err != nil { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } + return ctx.Err() +} + +// return the earliest STH timestamp within the right-most tile +func tileEarliestTimestamp(sths []*cttypes.SignedTreeHead) time.Time { + largestSTH, sths := sths[len(sths)-1], sths[:len(sths)-1] + tileNumber := largestSTH.TreeSize / ctclient.StaticTileWidth + earliest := largestSTH.TimestampTime() + for _, sth := range slices.Backward(sths) { + if sth.TreeSize/ctclient.StaticTileWidth != tileNumber { + break + } + if timestamp := sth.TimestampTime(); timestamp.Before(earliest) { + earliest = timestamp + } + } + return earliest +} + +func generateBatches(ctx context.Context, ctlog *loglist.Log, position uint64, number uint64, sths []*cttypes.SignedTreeHead, batches chan<- *batch) (uint64, uint64, error) { + downloadJobSize := downloadJobSize(ctlog) + if len(sths) == 0 { + return position, number, nil + } + largestSTH := sths[len(sths)-1] + treeSize := largestSTH.TreeSize + if ctlog.IsStaticCTAPI() && time.Since(tileEarliestTimestamp(sths)) < 5*time.Minute { + // Round down to the tile boundary to avoid downloading a partial tile that was recently discovered + // In a future invocation of this function, either enough time will have passed that this code path will be skipped, or the log will have grown and treeSize will be rounded to a larger tile boundary + treeSize -= treeSize % ctclient.StaticTileWidth + } + for { + batch := &batch{ + number: number, + begin: position, + end: min(treeSize, (position/downloadJobSize+1)*downloadJobSize), + } + for len(sths) > 0 && sths[0].TreeSize <= batch.end { + batch.sths = append(batch.sths, sths[0]) + sths = sths[1:] + } + select { + case <-ctx.Done(): + return position, number, ctx.Err() + default: + } + select { + case <-ctx.Done(): + return position, number, ctx.Err() + case batches <- batch: + } + number++ + if position == batch.end { + break + } + position = batch.end + } + return position, number, nil +} + +func downloadWorker(ctx context.Context, config *Config, ctlog *loglist.Log, client ctclient.Log, batchesIn <-chan *batch, batchesOut chan<- *batch) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + var batch *batch + select { + case <-ctx.Done(): + return ctx.Err() + case batch = <-batchesIn: + } + + entries, err := getEntriesFull(ctx, client, batch.begin, batch.end-1) + if err != nil { + return err + } + batch.entries = entries + + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + select { + case <-ctx.Done(): + return ctx.Err() + case batchesOut <- batch: + } + } + return nil +} + +func processWorker(ctx context.Context, config *Config, ctlog *loglist.Log, issuerGetter ctclient.IssuerGetter, batchesIn <-chan *batch, batchesOut *sequencer.Channel[batch]) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + var batch *batch + select { + case <-ctx.Done(): + return ctx.Err() + case batch = <-batchesIn: + } + for offset, entry := range batch.entries { + index := batch.begin + uint64(offset) + if err := processLogEntry(ctx, config, issuerGetter, &LogEntry{ + Entry: entry, + Index: index, + Log: ctlog, + }); err != nil { + return fmt.Errorf("error processing entry %d: %w", index, err) + } + } + if err := batchesOut.Add(ctx, batch.number, batch); err != nil { + return err + } + } +} + +func saveStateWorker(ctx context.Context, config *Config, ctlog *loglist.Log, state *LogState, batchesIn *sequencer.Channel[batch]) error { + for { + batch, err := batchesIn.Next(ctx) + if err != nil { + return err + } + if batch.begin != state.DownloadPosition.Size() { + panic(fmt.Errorf("saveStateWorker: expected batch to start at %d but got %d instead", state.DownloadPosition.Size(), batch.begin)) + } + rootHash := state.DownloadPosition.CalculateRoot() + for { + for len(batch.sths) > 0 && batch.sths[0].TreeSize == state.DownloadPosition.Size() { + sth := batch.sths[0] + batch.sths = batch.sths[1:] + if sth.RootHash != rootHash { + return &verifyEntriesError{ + sth: sth, + entriesRootHash: rootHash, + } + } + state.advanceVerifiedPosition() + state.VerifiedSTH = sth + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) + } + // don't remove the STH until state has been durably stored + if err := config.State.RemoveSTH(ctx, ctlog.LogID, sth); err != nil { + return fmt.Errorf("error removing verified STH: %w", err) + } + } + if len(batch.entries) == 0 { + break + } + entry := batch.entries[0] + batch.entries = batch.entries[1:] + leafHash := merkletree.HashLeaf(entry.LeafInput()) + state.DownloadPosition.Add(leafHash) + rootHash = state.DownloadPosition.CalculateRoot() + } + + if err := config.State.StoreLogState(ctx, ctlog.LogID, state); err != nil { + return fmt.Errorf("error storing log state: %w", err) + } + } +} + +func sleep(ctx context.Context, duration time.Duration) error { + timer := time.NewTimer(duration) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } } diff --git a/monitor/process.go b/monitor/process.go index c416d55..72b36fa 100644 --- a/monitor/process.go +++ b/monitor/process.go @@ -1,4 +1,4 @@ -// Copyright (C) 2023 Opsmate, Inc. +// Copyright (C) 2025 Opsmate, Inc. // // This Source Code Form is subject to the terms of the Mozilla // Public License, v. 2.0. If a copy of the MPL was not distributed @@ -10,79 +10,94 @@ package monitor import ( - "bytes" "context" "crypto/sha256" + "errors" "fmt" + "software.sslmate.com/src/certspotter" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/ctclient" + "software.sslmate.com/src/certspotter/cttypes" "software.sslmate.com/src/certspotter/loglist" - "software.sslmate.com/src/certspotter/merkletree" ) type LogEntry struct { - Log *loglist.Log - Index uint64 - LeafInput []byte - ExtraData []byte - LeafHash merkletree.Hash + ctclient.Entry + Index uint64 + Log *loglist.Log } -func processLogEntry(ctx context.Context, config *Config, entry *LogEntry) error { - leaf, err := ct.ReadMerkleTreeLeaf(bytes.NewReader(entry.LeafInput)) +func processLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry) error { + leaf, err := cttypes.ParseLeafInput(entry.LeafInput()) if err != nil { return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing Merkle Tree Leaf: %w", err)) } switch leaf.TimestampedEntry.EntryType { - case ct.X509LogEntryType: - return processX509LogEntry(ctx, config, entry, leaf.TimestampedEntry.X509Entry) - case ct.PrecertLogEntryType: - return processPrecertLogEntry(ctx, config, entry, leaf.TimestampedEntry.PrecertEntry) + case cttypes.X509EntryType: + return processX509LogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryASN1Cert) + case cttypes.PrecertEntryType: + return processPrecertLogEntry(ctx, config, issuerGetter, entry, leaf.TimestampedEntry.SignedEntryPreCert) default: return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("unknown log entry type %d", leaf.TimestampedEntry.EntryType)) } + return nil } -func processX509LogEntry(ctx context.Context, config *Config, entry *LogEntry, cert ct.ASN1Cert) error { - certInfo, err := certspotter.MakeCertInfoFromRawCert(cert) +func processX509LogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, cert *cttypes.ASN1Cert) error { + certInfo, err := certspotter.MakeCertInfoFromRawCert(*cert) if err != nil { return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing X.509 certificate: %w", err)) } - - chain, err := ct.UnmarshalX509ChainArray(entry.ExtraData) - if err != nil { - return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for X.509 entry: %w", err)) - } - chain = append([]ct.ASN1Cert{cert}, chain...) - if precertTBS, err := certspotter.ReconstructPrecertTBS(certInfo.TBS); err == nil { certInfo.TBS = precertTBS } else { return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error reconstructing precertificate TBSCertificate: %w", err)) } - return processCertificate(ctx, config, entry, certInfo, chain) + getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) { + var ( + chain = []cttypes.ASN1Cert{*cert} + errs = []error{} + ) + if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil { + chain = append(chain, issuers...) + } else { + errs = append(errs, err) + } + return chain, errors.Join(errs...) + } + return processCertificate(ctx, config, entry, certInfo, getChain) } -func processPrecertLogEntry(ctx context.Context, config *Config, entry *LogEntry, precert ct.PreCert) error { +func processPrecertLogEntry(ctx context.Context, config *Config, issuerGetter ctclient.IssuerGetter, entry *LogEntry, precert *cttypes.PreCert) error { certInfo, err := certspotter.MakeCertInfoFromRawTBS(precert.TBSCertificate) if err != nil { return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing precert TBSCertificate: %w", err)) } - - chain, err := ct.UnmarshalPrecertChainArray(entry.ExtraData) + precertBytes, err := entry.Precertificate() if err != nil { - return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error parsing extra_data for precert entry: %w", err)) + return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("error getting precert entry's precertificate: %w", err)) } - if _, err := certspotter.ValidatePrecert(chain[0], precert.TBSCertificate); err != nil { - return processMalformedLogEntry(ctx, config, entry, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err)) + getChain := func(ctx context.Context) ([]cttypes.ASN1Cert, error) { + var ( + chain = []cttypes.ASN1Cert{precertBytes} + errs = []error{} + ) + if issuers, err := entry.GetChain(ctx, issuerGetter); err == nil { + chain = append(chain, issuers...) + } else { + errs = append(errs, err) + } + if _, err := certspotter.ValidatePrecert(precertBytes, precert.TBSCertificate); err != nil { + errs = append(errs, fmt.Errorf("precertificate in extra_data does not match TBSCertificate in leaf_input: %w", err)) + } + return chain, errors.Join(errs...) } - - return processCertificate(ctx, config, entry, certInfo, chain) + return processCertificate(ctx, config, entry, certInfo, getChain) } -func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, chain []ct.ASN1Cert) error { +func processCertificate(ctx context.Context, config *Config, entry *LogEntry, certInfo *certspotter.CertInfo, getChain func(context.Context) ([]cttypes.ASN1Cert, error)) error { identifiers, err := certInfo.ParseIdentifiers() if err != nil { return processMalformedLogEntry(ctx, config, entry, err) @@ -92,11 +107,17 @@ func processCertificate(ctx context.Context, config *Config, entry *LogEntry, ce return nil } + chain, chainErr := getChain(ctx) + if errors.Is(chainErr, context.Canceled) { + return chainErr + } + cert := &DiscoveredCert{ WatchItem: watchItem, LogEntry: entry, Info: certInfo, Chain: chain, + ChainError: chainErr, TBSSHA256: sha256.Sum256(certInfo.TBS.Raw), SHA256: sha256.Sum256(chain[0]), PubkeySHA256: sha256.Sum256(certInfo.TBS.PublicKey.FullBytes), @@ -112,7 +133,7 @@ func processCertificate(ctx context.Context, config *Config, entry *LogEntry, ce func processMalformedLogEntry(ctx context.Context, config *Config, entry *LogEntry, parseError error) error { if err := config.State.NotifyMalformedEntry(ctx, entry, parseError); err != nil { - return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.URL, parseError, err) + return fmt.Errorf("error notifying about malformed log entry %d in %s (%q): %w", entry.Index, entry.Log.GetMonitoringURL(), parseError, err) } return nil } diff --git a/monitor/state.go b/monitor/state.go index a31e803..b08014a 100644 --- a/monitor/state.go +++ b/monitor/state.go @@ -11,17 +11,25 @@ package monitor import ( "context" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" "software.sslmate.com/src/certspotter/loglist" "software.sslmate.com/src/certspotter/merkletree" - "time" ) type LogState struct { DownloadPosition *merkletree.CollapsedTree `json:"download_position"` VerifiedPosition *merkletree.CollapsedTree `json:"verified_position"` - VerifiedSTH *ct.SignedTreeHead `json:"verified_sth"` - LastSuccess time.Time `json:"last_success"` + VerifiedSTH *cttypes.SignedTreeHead `json:"verified_sth"` +} + +func (state *LogState) rewindDownloadPosition() { + position := state.VerifiedPosition.Clone() + state.DownloadPosition = &position +} + +func (state *LogState) advanceVerifiedPosition() { + position := state.DownloadPosition.Clone() + state.VerifiedPosition = &position } // Methods are safe to call concurrently. @@ -43,14 +51,14 @@ type StateProvider interface { // Store STH for retrieval by LoadSTHs. If an STH with the same // timestamp and root hash is already stored, this STH can be ignored. - StoreSTH(context.Context, LogID, *ct.SignedTreeHead) error + StoreSTH(context.Context, LogID, *cttypes.SignedTreeHead) error // Load all STHs for this log previously stored with StoreSTH. // The returned slice must be sorted by tree size. - LoadSTHs(context.Context, LogID) ([]*ct.SignedTreeHead, error) + LoadSTHs(context.Context, LogID) ([]*cttypes.SignedTreeHead, error) // Remove an STH so it is no longer returned by LoadSTHs. - RemoveSTH(context.Context, LogID, *ct.SignedTreeHead) error + RemoveSTH(context.Context, LogID, *cttypes.SignedTreeHead) error // Called when a certificate matching the watch list is discovered. NotifyCert(context.Context, *DiscoveredCert) error diff --git a/monitor/statedir.go b/monitor/statedir.go index 9e3f4cb..5618a8c 100644 --- a/monitor/statedir.go +++ b/monitor/statedir.go @@ -16,11 +16,10 @@ import ( "io/fs" "os" "path/filepath" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" "software.sslmate.com/src/certspotter/merkletree" "strconv" "strings" - "time" ) func readVersion(stateDir string) (int, error) { @@ -50,7 +49,7 @@ func writeVersion(stateDir string) error { } func migrateLogStateDirV1(dir string) error { - var sth ct.SignedTreeHead + var sth cttypes.SignedTreeHead var tree merkletree.CollapsedTree sthPath := filepath.Join(dir, "sth.json") @@ -80,7 +79,6 @@ func migrateLogStateDirV1(dir string) error { DownloadPosition: &tree, VerifiedPosition: &tree, VerifiedSTH: &sth, - LastSuccess: time.Now().UTC(), } if err := writeJSONFile(filepath.Join(dir, "state.json"), stateFile, 0666); err != nil { return err diff --git a/monitor/sthdir.go b/monitor/sthdir.go index 2030dff..78ccc5a 100644 --- a/monitor/sthdir.go +++ b/monitor/sthdir.go @@ -21,19 +21,19 @@ import ( "os" "path/filepath" "slices" - "software.sslmate.com/src/certspotter/ct" + "software.sslmate.com/src/certspotter/cttypes" "strconv" "strings" ) -func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) { +func loadSTHsFromDir(dirPath string) ([]*cttypes.SignedTreeHead, error) { entries, err := os.ReadDir(dirPath) if errors.Is(err, fs.ErrNotExist) { - return []*ct.SignedTreeHead{}, nil + return []*cttypes.SignedTreeHead{}, nil } else if err != nil { return nil, err } - sths := make([]*ct.SignedTreeHead, 0, len(entries)) + sths := make([]*cttypes.SignedTreeHead, 0, len(entries)) for _, entry := range entries { filename := entry.Name() if strings.HasPrefix(filename, ".") || !strings.HasSuffix(filename, ".json") { @@ -45,23 +45,23 @@ func loadSTHsFromDir(dirPath string) ([]*ct.SignedTreeHead, error) { } sths = append(sths, sth) } - slices.SortFunc(sths, func(a, b *ct.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) }) + slices.SortFunc(sths, func(a, b *cttypes.SignedTreeHead) int { return cmp.Compare(a.TreeSize, b.TreeSize) }) return sths, nil } -func readSTHFile(filePath string) (*ct.SignedTreeHead, error) { +func readSTHFile(filePath string) (*cttypes.SignedTreeHead, error) { fileBytes, err := os.ReadFile(filePath) if err != nil { return nil, err } - sth := new(ct.SignedTreeHead) + sth := new(cttypes.SignedTreeHead) if err := json.Unmarshal(fileBytes, sth); err != nil { return nil, fmt.Errorf("error parsing %s: %w", filePath, err) } return sth, nil } -func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error { +func storeSTHInDir(dirPath string, sth *cttypes.SignedTreeHead) error { filePath := filepath.Join(dirPath, sthFilename(sth)) if fileExists(filePath) { return nil @@ -69,7 +69,7 @@ func storeSTHInDir(dirPath string, sth *ct.SignedTreeHead) error { return writeJSONFile(filePath, sth, 0666) } -func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error { +func removeSTHFromDir(dirPath string, sth *cttypes.SignedTreeHead) error { filePath := filepath.Join(dirPath, sthFilename(sth)) err := os.Remove(filePath) if err != nil && !errors.Is(err, fs.ErrNotExist) { @@ -79,15 +79,9 @@ func removeSTHFromDir(dirPath string, sth *ct.SignedTreeHead) error { } // generate a filename that uniquely identifies the STH (within the context of a particular log) -func sthFilename(sth *ct.SignedTreeHead) string { +func sthFilename(sth *cttypes.SignedTreeHead) string { hasher := sha256.New() - switch sth.Version { - case ct.V1: - binary.Write(hasher, binary.LittleEndian, sth.Timestamp) - binary.Write(hasher, binary.LittleEndian, sth.SHA256RootHash) - default: - panic(fmt.Errorf("sthFilename: invalid STH version %d", sth.Version)) - } - // For 6962-bis, we will need to handle a variable-length root hash, and include the signature in the filename hash (since signatures must be deterministic) + binary.Write(hasher, binary.LittleEndian, sth.Timestamp) + hasher.Write(sth.RootHash[:]) return strconv.FormatUint(sth.TreeSize, 10) + "-" + base64.RawURLEncoding.EncodeToString(hasher.Sum(nil)) + ".json" }