Compare commits

...

4 Commits

19 changed files with 394 additions and 39 deletions

View File

@ -108,7 +108,8 @@ func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, erro
"Accept-Language": {model.HTTPHeaderAcceptLanguage},
"User-Agent": {model.HTTPHeaderUserAgent},
},
TCPConnect: endpoints,
TCPConnect: endpoints,
XQUICEnabled: true,
}
data, err := json.Marshal(creq)
runtimex.PanicOnError(err, "oohelper: cannot marshal control request")

View File

@ -25,7 +25,7 @@ func analysisEngineClassic(tk *TestKeys, logger model.Logger) {
tk.analysisClassic(model.GeoIPASNLookupperFunc(geoipx.LookupASN), logger)
}
func (tk *TestKeys) analysisClassic(lookupper model.GeoIPASNLookupper, logger model.Logger) {
func (tk *TestKeys) analysisClassic(lookupper model.GeoIPASNLookupper, _ model.Logger) {
// Since we run after all tasks have completed (or so we assume) we're
// not going to use any form of locking here.

View File

@ -282,6 +282,9 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, network, address, a
}
if err == nil && httpRedirectIsRedirect(resp) {
err = httpValidateRedirect(resp)
if err == nil && t.FollowRedirects && !t.NumRedirects.CanFollowOneMoreRedirect() {
err = ErrTooManyRedirects
}
}
finished := trace.TimeSince(trace.ZeroTime())
@ -319,10 +322,7 @@ func (t *CleartextFlow) httpTransaction(ctx context.Context, network, address, a
// maybeFollowRedirects follows redirects if configured and needed
func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) {
if !t.FollowRedirects || !t.NumRedirects.CanFollowOneMoreRedirect() {
return // not configured or too many redirects
}
if httpRedirectIsRedirect(resp) {
if t.FollowRedirects && httpRedirectIsRedirect(resp) {
location, err := resp.Location()
if err != nil {
return // broken response from server

View File

@ -121,7 +121,7 @@ func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
Domain: URL.Hostname(),
IDGenerator: NewIDGenerator(),
Logger: sess.Logger(),
NumRedirects: NewNumRedirects(5),
NumRedirects: NewNumRedirects(10),
TestKeys: tk,
URL: URL,
ZeroTime: measurement.MeasurementStartTimeSaved,

View File

@ -19,5 +19,5 @@ func NewNumRedirects(n int64) *NumRedirects {
// CanFollowOneMoreRedirect returns true if we are
// allowed to follow one more redirect.
func (nr *NumRedirects) CanFollowOneMoreRedirect() bool {
return nr.count.Add(-1) > 0
return nr.count.Add(-1) >= 0
}

View File

@ -0,0 +1,16 @@
package webconnectivitylte
import "testing"
func TestNumRedirects(t *testing.T) {
const count = 10
nr := NewNumRedirects(count)
for idx := 0; idx < count; idx++ {
if !nr.CanFollowOneMoreRedirect() {
t.Fatal("got false with idx=", idx)
}
}
if nr.CanFollowOneMoreRedirect() {
t.Fatal("got true after the loop")
}
}

View File

@ -9,6 +9,7 @@ package webconnectivitylte
import (
"context"
"crypto/tls"
"errors"
"io"
"net"
"net/http"
@ -305,6 +306,9 @@ func (t *SecureFlow) newHTTPRequest(ctx context.Context) (*http.Request, error)
return httpReq, nil
}
// ErrTooManyRedirects indicates we have seen too many HTTP redirects.
var ErrTooManyRedirects = errors.New("stopped after too many redirects")
// httpTransaction runs the HTTP transaction and saves the results.
func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn string,
txp model.HTTPTransport, req *http.Request, trace *measurexlite.Trace) (*http.Response, []byte, error) {
@ -337,6 +341,9 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn
}
if err == nil && httpRedirectIsRedirect(resp) {
err = httpValidateRedirect(resp)
if err == nil && t.FollowRedirects && !t.NumRedirects.CanFollowOneMoreRedirect() {
err = ErrTooManyRedirects
}
}
finished := trace.TimeSince(trace.ZeroTime())
@ -374,10 +381,7 @@ func (t *SecureFlow) httpTransaction(ctx context.Context, network, address, alpn
// maybeFollowRedirects follows redirects if configured and needed
func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Response) {
if !t.FollowRedirects || !t.NumRedirects.CanFollowOneMoreRedirect() {
return // not configured or too many redirects
}
if httpRedirectIsRedirect(resp) {
if t.FollowRedirects && httpRedirectIsRedirect(resp) {
location, err := resp.Location()
if err != nil {
return // broken response from server

View File

@ -39,11 +39,29 @@ type connTrace struct {
var _ net.Conn = &connTrace{}
type remoteAddrProvider interface {
RemoteAddr() net.Addr
}
func safeRemoteAddrNetwork(rap remoteAddrProvider) (result string) {
if addr := rap.RemoteAddr(); addr != nil {
result = addr.Network()
}
return result
}
func safeRemoteAddrString(rap remoteAddrProvider) (result string) {
if addr := rap.RemoteAddr(); addr != nil {
result = addr.String()
}
return result
}
// Read implements net.Conn.Read and saves network events.
func (c *connTrace) Read(b []byte) (int, error) {
// collect preliminary stats when the connection is surely active
network := c.RemoteAddr().Network()
addr := c.RemoteAddr().String()
network := safeRemoteAddrNetwork(c)
addr := safeRemoteAddrString(c)
started := c.tx.TimeSince(c.tx.ZeroTime())
// perform the underlying network operation
@ -99,8 +117,8 @@ func (tx *Trace) CloneBytesReceivedMap() (out map[string]int64) {
// Write implements net.Conn.Write and saves network events.
func (c *connTrace) Write(b []byte) (int, error) {
network := c.RemoteAddr().Network()
addr := c.RemoteAddr().String()
network := safeRemoteAddrNetwork(c)
addr := safeRemoteAddrString(c)
started := c.tx.TimeSince(c.tx.ZeroTime())
count, err := c.Conn.Write(b)

View File

@ -12,6 +12,43 @@ import (
"github.com/ooni/probe-cli/v3/internal/testingx"
)
func TestRemoteAddrProvider(t *testing.T) {
t.Run("for nil address", func(t *testing.T) {
conn := &mocks.Conn{
MockRemoteAddr: func() net.Addr {
return nil
},
}
if safeRemoteAddrNetwork(conn) != "" {
t.Fatal("expected empty network")
}
if safeRemoteAddrString(conn) != "" {
t.Fatal("expected empty string")
}
})
t.Run("for common case", func(t *testing.T) {
conn := &mocks.Conn{
MockRemoteAddr: func() net.Addr {
return &mocks.Addr{
MockString: func() string {
return "1.1.1.1:443"
},
MockNetwork: func() string {
return "tcp"
},
}
},
}
if safeRemoteAddrNetwork(conn) != "tcp" {
t.Fatal("unexpected network")
}
if safeRemoteAddrString(conn) != "1.1.1.1:443" {
t.Fatal("unexpected string")
}
})
}
func TestMaybeClose(t *testing.T) {
t.Run("with nil conn", func(t *testing.T) {
var conn net.Conn = nil

View File

@ -1,8 +1,11 @@
package netemx
import (
"fmt"
"net"
"net/http"
"strconv"
"strings"
"github.com/ooni/netem"
)
@ -29,7 +32,7 @@ func HTTPBinHandlerFactory() HTTPHandlerFactory {
// Any other request URL causes a 404 respose.
func HTTPBinHandler() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Date", "Thu, 24 Aug 2023 14:35:29 GMT")
w.Header().Set("Date", "Thu, 24 Aug 2023 14:35:29 GMT")
// missing address => 500
address, _, err := net.SplitHostPort(r.RemoteAddr)
@ -44,6 +47,20 @@ func HTTPBinHandler() http.Handler {
secureRedirect := r.URL.Path == "/broken-redirect-https"
switch {
// redirect with count
case strings.HasPrefix(r.URL.Path, "/redirect/"):
count, err := strconv.Atoi(strings.TrimPrefix(r.URL.Path, "/redirect/"))
if err != nil {
w.WriteHeader(http.StatusBadRequest)
return
}
if count <= 0 {
w.WriteHeader(http.StatusOK)
return
}
w.Header().Set("Location", fmt.Sprintf("/redirect/%d", count-1))
w.WriteHeader(http.StatusFound)
// broken HTTP redirect for clients
case cleartextRedirect && client:
w.Header().Set("Location", "http://")

View File

@ -25,6 +25,88 @@ func TestHTTPBinHandler(t *testing.T) {
}
})
t.Run("/redirect/{n} with invalid number", func(t *testing.T) {
req := &http.Request{
URL: &url.URL{Scheme: "https://", Path: "/redirect/antani"},
Body: http.NoBody,
Close: false,
Host: "httpbin.com",
RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
}
rr := httptest.NewRecorder()
handler := HTTPBinHandler()
handler.ServeHTTP(rr, req)
result := rr.Result()
if result.StatusCode != http.StatusBadRequest {
t.Fatal("unexpected status code", result.StatusCode)
}
})
t.Run("/redirect/0", func(t *testing.T) {
req := &http.Request{
URL: &url.URL{Scheme: "https://", Path: "/redirect/0"},
Body: http.NoBody,
Close: false,
Host: "httpbin.com",
RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
}
rr := httptest.NewRecorder()
handler := HTTPBinHandler()
handler.ServeHTTP(rr, req)
result := rr.Result()
if result.StatusCode != http.StatusOK {
t.Fatal("unexpected status code", result.StatusCode)
}
})
t.Run("/redirect/1", func(t *testing.T) {
req := &http.Request{
URL: &url.URL{Scheme: "https://", Path: "/redirect/1"},
Body: http.NoBody,
Close: false,
Host: "httpbin.com",
RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
}
rr := httptest.NewRecorder()
handler := HTTPBinHandler()
handler.ServeHTTP(rr, req)
result := rr.Result()
if result.StatusCode != http.StatusFound {
t.Fatal("unexpected status code", result.StatusCode)
}
location, err := result.Location()
if err != nil {
t.Fatal(err)
}
if location.Path != "/redirect/0" {
t.Fatal("unexpected location.Path", location.Path)
}
})
t.Run("/redirect/2", func(t *testing.T) {
req := &http.Request{
URL: &url.URL{Scheme: "https://", Path: "/redirect/2"},
Body: http.NoBody,
Close: false,
Host: "httpbin.com",
RemoteAddr: net.JoinHostPort("8.8.8.8", "54321"),
}
rr := httptest.NewRecorder()
handler := HTTPBinHandler()
handler.ServeHTTP(rr, req)
result := rr.Result()
if result.StatusCode != http.StatusFound {
t.Fatal("unexpected status code", result.StatusCode)
}
location, err := result.Location()
if err != nil {
t.Fatal(err)
}
if location.Path != "/redirect/1" {
t.Fatal("unexpected location.Path", location.Path)
}
})
t.Run("/broken-redirect-http with client address", func(t *testing.T) {
req := &http.Request{
URL: &url.URL{Scheme: "http://", Path: "/broken-redirect-http"},

View File

@ -35,7 +35,7 @@ func (p *OOAPIHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
}
func (p *OOAPIHandler) getApiV1TestHelpers(w http.ResponseWriter, r *http.Request) {
func (p *OOAPIHandler) getApiV1TestHelpers(w http.ResponseWriter, _ *http.Request) {
resp := map[string][]model.OOAPIService{
"web-connectivity": {
{

View File

@ -73,13 +73,7 @@ func TestOOHelperDHandler(t *testing.T) {
Failure: nil,
},
},
QUICHandshake: map[string]model.THTLSHandshakeResult{
"93.184.216.34:443": {
ServerName: "www.example.com",
Status: true,
Failure: nil,
},
},
QUICHandshake: map[string]model.THTLSHandshakeResult{}, // since https://github.com/ooni/probe-cli/pull/1549
HTTPRequest: model.THHTTPRequestResult{
BodyLength: 1533,
DiscoveredH3Endpoint: "www.example.com:443",
@ -93,19 +87,7 @@ func TestOOHelperDHandler(t *testing.T) {
},
StatusCode: 200,
},
HTTP3Request: &model.THHTTPRequestResult{
BodyLength: 1533,
DiscoveredH3Endpoint: "",
Failure: nil,
Title: "Default Web Page",
Headers: map[string]string{
"Alt-Svc": `h3=":443"`,
"Content-Length": "1533",
"Content-Type": "text/html; charset=utf-8",
"Date": "Thu, 24 Aug 2023 14:35:29 GMT",
},
StatusCode: 200,
},
HTTP3Request: nil, // since https://github.com/ooni/probe-cli/pull/1549
DNS: model.THDNSResult{
Failure: nil,
Addrs: []string{"93.184.216.34"},

View File

@ -11,6 +11,7 @@ import (
"io"
"net/http"
"net/http/cookiejar"
"os"
"strings"
"sync/atomic"
"time"
@ -31,6 +32,9 @@ const maxAcceptableBodySize = 1 << 24
//
// The zero value is invalid; construct using [NewHandler].
type Handler struct {
// EnableQUIC OPTIONALLY enables QUIC.
EnableQUIC bool
// baseLogger is the MANDATORY logger to use.
baseLogger model.Logger
@ -69,9 +73,13 @@ type Handler struct {
var _ http.Handler = &Handler{}
// enableQUIC allows to control whether to enable QUIC by using environment variables.
var enableQUIC = (os.Getenv("OOHELPERD_ENABLE_QUIC") == "1")
// NewHandler constructs the [handler].
func NewHandler(logger model.Logger, netx *netxlite.Netx) *Handler {
return &Handler{
EnableQUIC: enableQUIC,
baseLogger: logger,
countRequests: &atomic.Int64{},
indexer: &atomic.Int64{},

View File

@ -262,3 +262,10 @@ func TestHandlerWorkingAsIntended(t *testing.T) {
})
}
}
func TestNewHandlerEnableQUIC(t *testing.T) {
handler := NewHandler(log.Log, &netxlite.Netx{Underlying: nil})
if handler.EnableQUIC != false {
t.Fatal("expected to see false here (is the the environment variable OOHELPERD_ENABLE_QUIC set?!)")
}
}

View File

@ -125,7 +125,7 @@ func measure(ctx context.Context, config *Handler, creq *ctrlRequest) (*ctrlResp
// In the v3.17.x and possibly v3.18.x release cycles, QUIC is disabled by
// default but clients that know QUIC can enable it. We will eventually remove
// this flag and enable QUIC measurements for all clients.
if creq.XQUICEnabled && cresp.HTTPRequest.DiscoveredH3Endpoint != "" {
if config.EnableQUIC && creq.XQUICEnabled && cresp.HTTPRequest.DiscoveredH3Endpoint != "" {
// quicconnect: start over all the endpoints
for _, endpoint := range endpoints {
wg.Add(1)

View File

@ -0,0 +1,137 @@
package oohelperd_test
import (
"bytes"
"context"
"net/http"
"net/http/httptest"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/must"
"github.com/ooni/probe-cli/v3/internal/netemx"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/oohelperd"
"github.com/ooni/probe-cli/v3/internal/optional"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
// TestQAEnableDisableQUIC ensures that we can enable and disable QUIC.
func TestQAEnableDisableQUIC(t *testing.T) {
// testcase is a test case for this function
type testcase struct {
name string
enableQUIC optional.Value[bool]
}
cases := []testcase{{
name: "with the default settings",
enableQUIC: optional.None[bool](),
}, {
name: "with explicit false",
enableQUIC: optional.Some(false),
}, {
name: "with explicit true",
enableQUIC: optional.Some(true),
}}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
// create a new testing scenario
env := netemx.MustNewScenario(netemx.InternetScenario)
defer env.Close()
// create a new handler
handler := oohelperd.NewHandler(
log.Log,
&netxlite.Netx{Underlying: &netxlite.NetemUnderlyingNetworkAdapter{UNet: env.ClientStack}},
)
// optionally and conditionally enable QUIC
if !tc.enableQUIC.IsNone() {
handler.EnableQUIC = tc.enableQUIC.Unwrap()
}
// create request body
reqbody := &model.THRequest{
HTTPRequest: "https://www.example.com/",
HTTPRequestHeaders: map[string][]string{
"Accept-Language": {model.HTTPHeaderAcceptLanguage},
"Accept": {model.HTTPHeaderAccept},
"User-Agent": {model.HTTPHeaderUserAgent},
},
TCPConnect: []string{netemx.AddressWwwExampleCom},
XQUICEnabled: true,
}
// create request
req := runtimex.Try1(http.NewRequest(
"POST",
"http://127.0.0.1:8080/",
bytes.NewReader(must.MarshalJSON(reqbody)),
))
// create response recorder
resprec := httptest.NewRecorder()
// invoke the handler
handler.ServeHTTP(resprec, req)
// get the response
resp := resprec.Result()
defer resp.Body.Close()
// make sure the status code indicates success
if resp.StatusCode != 200 {
t.Fatal("expected 200 Ok")
}
// make sure the content-type is OK
if v := resp.Header.Get("Content-Type"); v != "application/json" {
t.Fatal("unexpected content-type", v)
}
// read the response body
respbody := runtimex.Try1(netxlite.ReadAllContext(context.Background(), resp.Body))
// parse the response body
var jsonresp model.THResponse
must.UnmarshalJSON(respbody, &jsonresp)
// check whether we have an HTTP3 response
switch {
case !tc.enableQUIC.IsNone() && tc.enableQUIC.Unwrap() && jsonresp.HTTP3Request != nil:
// all good: we have QUIC enabled and we get an HTTP/3 response
case (tc.enableQUIC.IsNone() || tc.enableQUIC.Unwrap() == false) && jsonresp.HTTP3Request == nil:
// all good: either default behavior or QUIC not enabled and not HTTP/3 response
default:
t.Fatalf(
"tc.enableQUIC.IsNone() = %v, tc.enableQUIC.UnwrapOr(false) = %v, jsonresp.HTTP3Request = %v",
tc.enableQUIC.IsNone(),
tc.enableQUIC.UnwrapOr(false),
jsonresp.HTTP3Request,
)
}
// check whether we have QUIC handshakes
switch {
case !tc.enableQUIC.IsNone() && tc.enableQUIC.Unwrap() && len(jsonresp.QUICHandshake) > 0:
// all good: we have QUIC enabled and we get QUIC handshakes
case (tc.enableQUIC.IsNone() || tc.enableQUIC.Unwrap() == false) && len(jsonresp.QUICHandshake) <= 0:
// all good: either default behavior or QUIC not enabled and no QUIC handshakes
default:
t.Fatalf(
"tc.enableQUIC.IsNone() = %v, tc.enableQUIC.UnwrapOr(false) = %v, jsonresp.QUICHandshake = %v",
tc.enableQUIC.IsNone(),
tc.enableQUIC.UnwrapOr(false),
jsonresp.QUICHandshake,
)
}
})
}
}

View File

@ -391,3 +391,47 @@ func redirectWithBrokenLocationForHTTPS() *TestCase {
},
}
}
// redirectWithMoreThanTenRedirectsAndHTTP is a scenario where the redirect
// consists of more than ten redirects for http:// URLs.
func redirectWithMoreThanTenRedirectsAndHTTP() *TestCase {
return &TestCase{
Name: "redirectWithMoreThanTenRedirectsAndHTTP",
Flags: TestCaseFlagNoV04,
Input: "http://httpbin.com/redirect/15",
LongTest: true,
ExpectErr: false,
ExpectTestKeys: &TestKeys{
DNSExperimentFailure: nil,
DNSConsistency: "consistent",
HTTPExperimentFailure: `unknown_failure: stopped after too many redirects`,
XStatus: 0,
XDNSFlags: 0,
XBlockingFlags: 0,
Accessible: false,
Blocking: false,
},
}
}
// redirectWithMoreThanTenRedirectsAndHTTPS is a scenario where the redirect
// consists of more than ten redirects for https:// URLs.
func redirectWithMoreThanTenRedirectsAndHTTPS() *TestCase {
return &TestCase{
Name: "redirectWithMoreThanTenRedirectsAndHTTPS",
Flags: TestCaseFlagNoV04,
Input: "https://httpbin.com/redirect/15",
LongTest: true,
ExpectErr: false,
ExpectTestKeys: &TestKeys{
DNSExperimentFailure: nil,
DNSConsistency: "consistent",
HTTPExperimentFailure: `unknown_failure: stopped after too many redirects`,
XStatus: 0,
XDNSFlags: 0,
XBlockingFlags: 0,
Accessible: false,
Blocking: false,
},
}
}

View File

@ -90,6 +90,8 @@ func AllTestCases() []*TestCase {
redirectWithConsistentDNSAndThenEOFForHTTPS(),
redirectWithConsistentDNSAndThenTimeoutForHTTP(),
redirectWithConsistentDNSAndThenTimeoutForHTTPS(),
redirectWithMoreThanTenRedirectsAndHTTP(),
redirectWithMoreThanTenRedirectsAndHTTPS(),
successWithHTTP(),
successWithHTTPS(),