From 2a2088f7df71269e5bec1116e66a903600153e2c Mon Sep 17 00:00:00 2001 From: dprotaso Date: Thu, 12 Oct 2023 12:56:41 -0400 Subject: [PATCH 1/4] GEP-1911 - h2c backend protocol --- conformance/base/manifests.yaml | 8 +- .../tests/httproute-backend-protocol-h2c.go | 83 +++++++++++++++++++ .../tests/httproute-backend-protocol-h2c.yaml | 17 ++++ conformance/utils/http/http.go | 6 +- .../utils/roundtripper/roundtripper.go | 74 +++++++++++++---- conformance/utils/suite/features.go | 4 + 6 files changed, 176 insertions(+), 16 deletions(-) create mode 100644 conformance/tests/httproute-backend-protocol-h2c.go create mode 100644 conformance/tests/httproute-backend-protocol-h2c.yaml diff --git a/conformance/base/manifests.yaml b/conformance/base/manifests.yaml index f3b571ee35..361797c65c 100644 --- a/conformance/base/manifests.yaml +++ b/conformance/base/manifests.yaml @@ -91,9 +91,15 @@ spec: selector: app: infra-backend-v1 ports: - - protocol: TCP + - name: first-port + protocol: TCP port: 8080 targetPort: 3000 + - name: second-port + protocol: TCP + appProtocol: kubernetes.io/h2c + port: 8081 + targetPort: 3001 --- apiVersion: apps/v1 kind: Deployment diff --git a/conformance/tests/httproute-backend-protocol-h2c.go b/conformance/tests/httproute-backend-protocol-h2c.go new file mode 100644 index 0000000000..2e620d95bb --- /dev/null +++ b/conformance/tests/httproute-backend-protocol-h2c.go @@ -0,0 +1,83 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tests + +import ( + "testing" + + "k8s.io/apimachinery/pkg/types" + + "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/roundtripper" + "sigs.k8s.io/gateway-api/conformance/utils/suite" +) + +func init() { + ConformanceTests = append(ConformanceTests, HTTPRouteBackendProtocolH2C) +} + +var HTTPRouteBackendProtocolH2C = suite.ConformanceTest{ + ShortName: "HTTPRouteBackendProtocolH2C", + Description: "A HTTPRoute with a BackendRef that has an appProtocol kubernetes.io/h2c should be functional", + Features: []suite.SupportedFeature{ + suite.SupportGateway, + suite.SupportHTTPRoute, + suite.SupportHTTPRouteBackendProtocolH2C, + }, + Manifests: []string{ + "tests/httproute-backend-protocol-h2c.yaml", + }, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "backend-protocol-h2c", Namespace: ns} + gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + + // TODO - client h2c upgrade flow + // + // Go's HTTP client is unable to handle the protocol change transparently see: https://github.com/golang/go/issues/46249 + // + // t.Run("h2c upgrade request should reach backend", func(t *testing.T) { + // http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, http.ExpectedResponse{ + // Request: http.Request{ + // Path: "/", + // Headers: map[string]string{ + // "Connection": "Upgrade, HTTP2-Settings", + // "Upgrade": "h2c", + // "HTTP2-Settings": "", + // }, + // }, + // Response: http.Response{StatusCode: 200}, + // Backend: "infra-backend-v1", + // Namespace: "gateway-conformance-infra", + // }) + // }) + + t.Run("http2 prior knowledge request should reach backend", func(t *testing.T) { + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, http.ExpectedResponse{ + Request: http.Request{ + Path: "/", + Protocol: roundtripper.H2CPriorKnowledgeProtocol, + }, + Response: http.Response{StatusCode: 200}, + Backend: "infra-backend-v1", + Namespace: "gateway-conformance-infra", + }) + }) + }, +} diff --git a/conformance/tests/httproute-backend-protocol-h2c.yaml b/conformance/tests/httproute-backend-protocol-h2c.yaml new file mode 100644 index 0000000000..12106973f7 --- /dev/null +++ b/conformance/tests/httproute-backend-protocol-h2c.yaml @@ -0,0 +1,17 @@ +apiVersion: gateway.networking.k8s.io/v1beta1 +kind: HTTPRoute +metadata: + name: backend-protocol-h2c + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + rules: + - backendRefs: + # This points to a Service with the following ServicePort + # - protocol: TCP + # appProtocol: kubernetes.io/h2c + # port: 8081 + # targetPort: 3001 + - name: infra-backend-v1 + port: 8081 diff --git a/conformance/utils/http/http.go b/conformance/utils/http/http.go index 784696d391..d63cb5fcec 100644 --- a/conformance/utils/http/http.go +++ b/conformance/utils/http/http.go @@ -116,6 +116,10 @@ func MakeRequest(t *testing.T, expected *ExpectedResponse, gwAddr, protocol, sch expected.Response.StatusCode = 200 } + if expected.Request.Protocol == "" { + expected.Request.Protocol = protocol + } + path, query, _ := strings.Cut(expected.Request.Path, "?") reqURL := url.URL{Scheme: scheme, Host: CalculateHost(t, gwAddr, scheme), Path: path, RawQuery: query} @@ -125,7 +129,7 @@ func MakeRequest(t *testing.T, expected *ExpectedResponse, gwAddr, protocol, sch Method: expected.Request.Method, Host: expected.Request.Host, URL: reqURL, - Protocol: protocol, + Protocol: expected.Request.Protocol, Headers: map[string][]string{}, UnfollowRedirect: expected.Request.UnfollowRedirect, } diff --git a/conformance/utils/roundtripper/roundtripper.go b/conformance/utils/roundtripper/roundtripper.go index efeeab3e2b..b2d21487fb 100644 --- a/conformance/utils/roundtripper/roundtripper.go +++ b/conformance/utils/roundtripper/roundtripper.go @@ -21,6 +21,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/json" + "errors" "fmt" "io" "net" @@ -29,9 +30,15 @@ import ( "net/url" "regexp" + "golang.org/x/net/http2" + "sigs.k8s.io/gateway-api/conformance/utils/config" ) +const ( + H2CPriorKnowledgeProtocol = "H2C_PRIOR_KNOWLEDGE" +) + // RoundTripper is an interface used to make requests within conformance tests. // This can be overridden with custom implementations whenever necessary. type RoundTripper interface { @@ -104,19 +111,7 @@ type DefaultRoundTripper struct { CustomDialContext func(context.Context, string, string) (net.Conn, error) } -// CaptureRoundTrip makes a request with the provided parameters and returns the -// captured request and response from echoserver. An error will be returned if -// there is an error running the function but not if an HTTP error status code -// is received. -func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedRequest, *CapturedResponse, error) { - client := &http.Client{} - - if request.UnfollowRedirect { - client.CheckRedirect = func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - } - } - +func (d *DefaultRoundTripper) httpTransport(request Request) (http.RoundTripper, error) { transport := &http.Transport{ DialContext: d.CustomDialContext, // We disable keep-alives so that we don't leak established TCP connections. @@ -131,10 +126,61 @@ func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedReques if request.Server != "" && len(request.CertPem) != 0 && len(request.KeyPem) != 0 { tlsConfig, err := tlsClientConfig(request.Server, request.CertPem, request.KeyPem) if err != nil { - return nil, nil, err + return nil, err } transport.TLSClientConfig = tlsConfig } + + return transport, nil +} + +func (d *DefaultRoundTripper) h2cPriorKnowledgeTransport(request Request) (http.RoundTripper, error) { + if request.Server != "" && len(request.CertPem) != 0 && len(request.KeyPem) != 0 { + return nil, errors.New("request has configured cert and key but h2 prior knowledge is not encrypted") + } + + transport := &http2.Transport{ + AllowHTTP: true, + DialTLSContext: func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, network, addr) + }, + } + + return transport, nil +} + +// CaptureRoundTrip makes a request with the provided parameters and returns the +// captured request and response from echoserver. An error will be returned if +// there is an error running the function but not if an HTTP error status code +// is received. +func (d *DefaultRoundTripper) CaptureRoundTrip(request Request) (*CapturedRequest, *CapturedResponse, error) { + var transport http.RoundTripper + var err error + + switch request.Protocol { + case H2CPriorKnowledgeProtocol: + transport, err = d.h2cPriorKnowledgeTransport(request) + default: + transport, err = d.httpTransport(request) + } + + if err != nil { + return nil, nil, err + } + + return d.defaultRoundTrip(request, transport) +} + +func (d *DefaultRoundTripper) defaultRoundTrip(request Request, transport http.RoundTripper) (*CapturedRequest, *CapturedResponse, error) { + client := &http.Client{} + + if request.UnfollowRedirect { + client.CheckRedirect = func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + } + } + client.Transport = transport method := "GET" diff --git a/conformance/utils/suite/features.go b/conformance/utils/suite/features.go index cf687c33ec..62a7bca734 100644 --- a/conformance/utils/suite/features.go +++ b/conformance/utils/suite/features.go @@ -133,6 +133,9 @@ const ( // This option indicates support for HTTPRoute backendRequest timeouts (extended conformance). SupportHTTPRouteBackendTimeout SupportedFeature = "HTTPRouteBackendTimeout" + + // This option indicates support for HTTPRoute with a backendref with an appProtoocol 'kubernetes.io/h2c' + SupportHTTPRouteBackendProtocolH2C SupportedFeature = "HTTPRouteBackendProtocolH2C" ) // HTTPRouteExtendedFeatures includes all the supported features for HTTPRoute @@ -167,6 +170,7 @@ const ( // Implementations have the flexibility to opt-in for either specific features or the entire set. var HTTPRouteExperimentalFeatures = sets.New( SupportHTTPRouteDestinationPortMatching, + SupportHTTPRouteBackendProtocolH2C, ) // ----------------------------------------------------------------------------- From 6ebfc327dd3f2b70be586f1a709e6df41ac9fdc2 Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Mon, 16 Oct 2023 09:20:27 -0400 Subject: [PATCH 2/4] h2c upgrade mechanism is deprecated --- .../tests/httproute-backend-protocol-h2c.go | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/conformance/tests/httproute-backend-protocol-h2c.go b/conformance/tests/httproute-backend-protocol-h2c.go index 2e620d95bb..2a4baedc2a 100644 --- a/conformance/tests/httproute-backend-protocol-h2c.go +++ b/conformance/tests/httproute-backend-protocol-h2c.go @@ -48,25 +48,8 @@ var HTTPRouteBackendProtocolH2C = suite.ConformanceTest{ gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns} gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) - // TODO - client h2c upgrade flow - // - // Go's HTTP client is unable to handle the protocol change transparently see: https://github.com/golang/go/issues/46249 - // - // t.Run("h2c upgrade request should reach backend", func(t *testing.T) { - // http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, http.ExpectedResponse{ - // Request: http.Request{ - // Path: "/", - // Headers: map[string]string{ - // "Connection": "Upgrade, HTTP2-Settings", - // "Upgrade": "h2c", - // "HTTP2-Settings": "", - // }, - // }, - // Response: http.Response{StatusCode: 200}, - // Backend: "infra-backend-v1", - // Namespace: "gateway-conformance-infra", - // }) - // }) + // We are not testing the h2c HTTP upgrade mechanism as it is deprecated + // See: https://datatracker.ietf.org/doc/html/rfc9113#versioning t.Run("http2 prior knowledge request should reach backend", func(t *testing.T) { http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, http.ExpectedResponse{ From c9a0b0ecf86068e9d1bdde3446f19aa69a1676b0 Mon Sep 17 00:00:00 2001 From: dprotaso Date: Tue, 17 Oct 2023 10:15:20 -0400 Subject: [PATCH 3/4] use v1 API --- conformance/tests/httproute-backend-protocol-h2c.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/tests/httproute-backend-protocol-h2c.yaml b/conformance/tests/httproute-backend-protocol-h2c.yaml index 12106973f7..7b20c347ac 100644 --- a/conformance/tests/httproute-backend-protocol-h2c.yaml +++ b/conformance/tests/httproute-backend-protocol-h2c.yaml @@ -1,4 +1,4 @@ -apiVersion: gateway.networking.k8s.io/v1beta1 +apiVersion: gateway.networking.k8s.io/v1 kind: HTTPRoute metadata: name: backend-protocol-h2c From 03674788667e687fdebed6f1b9366764be35f379 Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Tue, 24 Oct 2023 09:39:22 -0400 Subject: [PATCH 4/4] fix typo --- conformance/utils/suite/features.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conformance/utils/suite/features.go b/conformance/utils/suite/features.go index 62a7bca734..063920f68a 100644 --- a/conformance/utils/suite/features.go +++ b/conformance/utils/suite/features.go @@ -134,7 +134,7 @@ const ( // This option indicates support for HTTPRoute backendRequest timeouts (extended conformance). SupportHTTPRouteBackendTimeout SupportedFeature = "HTTPRouteBackendTimeout" - // This option indicates support for HTTPRoute with a backendref with an appProtoocol 'kubernetes.io/h2c' + // This option indicates support for HTTPRoute with a backendref with an appProtocol 'kubernetes.io/h2c' SupportHTTPRouteBackendProtocolH2C SupportedFeature = "HTTPRouteBackendProtocolH2C" )