// Copyright (C) 2020, 2023 Opsmate, Inc. // // This Source Code Form is subject to the terms of the Mozilla // Public License, v. 2.0. If a copy of the MPL was not distributed // with this file, You can obtain one at http://mozilla.org/MPL/2.0/. // // This software is distributed WITHOUT A WARRANTY OF ANY KIND. // See the Mozilla Public License for details. package loglist import ( "context" "encoding/json" "errors" "fmt" "io" "net/http" "os" "strings" "time" ) var UserAgent = "certspotter" type ModificationToken struct { etag string modified time.Time } var ErrNotModified = errors.New("loglist has not been modified") func newModificationToken(response *http.Response) *ModificationToken { token := &ModificationToken{ etag: response.Header.Get("ETag"), } if t, err := time.Parse(http.TimeFormat, response.Header.Get("Last-Modified")); err == nil { token.modified = t } return token } func (token *ModificationToken) setRequestHeaders(request *http.Request) { if token.etag != "" { request.Header.Set("If-None-Match", token.etag) } else if !token.modified.IsZero() { request.Header.Set("If-Modified-Since", token.modified.Format(http.TimeFormat)) } } func Load(ctx context.Context, urlOrFile string) (*List, error) { list, _, err := LoadIfModified(ctx, urlOrFile, nil) return list, err } func LoadIfModified(ctx context.Context, urlOrFile string, token *ModificationToken) (*List, *ModificationToken, error) { if strings.HasPrefix(urlOrFile, "https://") { return FetchIfModified(ctx, urlOrFile, token) } else { list, err := ReadFile(urlOrFile) return list, nil, err } } func Fetch(ctx context.Context, url string) (*List, error) { list, _, err := FetchIfModified(ctx, url, nil) return list, err } func FetchIfModified(ctx context.Context, url string, token *ModificationToken) (*List, *ModificationToken, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, nil, err } request.Header.Set("User-Agent", UserAgent) if token != nil { token.setRequestHeaders(request) } response, err := http.DefaultClient.Do(request) if err != nil { return nil, nil, err } content, err := io.ReadAll(response.Body) response.Body.Close() if err != nil { return nil, nil, err } if token != nil && response.StatusCode == http.StatusNotModified { return nil, nil, ErrNotModified } if response.StatusCode != 200 { return nil, nil, fmt.Errorf("%s: %s", url, response.Status) } list, err := Unmarshal(content) if err != nil { return nil, nil, fmt.Errorf("error parsing %s: %w", url, err) } return list, newModificationToken(response), err } func ReadFile(filename string) (*List, error) { content, err := os.ReadFile(filename) if err != nil { return nil, err } return Unmarshal(content) } func Unmarshal(jsonBytes []byte) (*List, error) { list := new(List) if err := json.Unmarshal(jsonBytes, list); err != nil { return nil, err } if err := list.Validate(); err != nil { return nil, fmt.Errorf("Invalid log list: %s", err) } return list, nil }