diff --git a/pkg/asset/manifests/gcp/cluster.go b/pkg/asset/manifests/gcp/cluster.go index 347ac22f19b..5f686ef62cb 100644 --- a/pkg/asset/manifests/gcp/cluster.go +++ b/pkg/asset/manifests/gcp/cluster.go @@ -16,6 +16,7 @@ import ( "github.com/openshift/installer/pkg/asset/installconfig" "github.com/openshift/installer/pkg/asset/manifests/capiutils" gcpconsts "github.com/openshift/installer/pkg/constants/gcp" + "github.com/openshift/installer/pkg/types" "github.com/openshift/installer/pkg/types/gcp" ) @@ -101,6 +102,11 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID labels[label.Key] = label.Value } + capgLoadBalancerType := capg.InternalExternal + if installConfig.Config.Publish == types.InternalPublishingStrategy { + capgLoadBalancerType = capg.Internal + } + gcpCluster := &capg.GCPCluster{ ObjectMeta: metav1.ObjectMeta{ Name: clusterID.InfraID, @@ -121,6 +127,7 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID FailureDomains: findFailureDomains(installConfig), LoadBalancer: capg.LoadBalancerSpec{ APIServerInstanceGroupTagOverride: ptr.To(InstanceGroupRoleTag), + LoadBalancerType: ptr.To(capgLoadBalancerType), }, }, } diff --git a/pkg/infrastructure/gcp/clusterapi/clusterapi.go b/pkg/infrastructure/gcp/clusterapi/clusterapi.go index 11278718f18..dcf0fe00d5d 100644 --- a/pkg/infrastructure/gcp/clusterapi/clusterapi.go +++ b/pkg/infrastructure/gcp/clusterapi/clusterapi.go @@ -186,17 +186,6 @@ func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput) return fmt.Errorf("could not find master subnet %s in subnets %v", masterSubnetName, subnets) } - zones := gcpCluster.Status.FailureDomains.GetIDs() - - // Currently, the internal/private load balancer is not created by CAPG. The load balancer will be created - // by the installer for now. - // TODO: remove the creation of the LB and health check here when supported by CAPG. - // https://github.com/kubernetes-sigs/cluster-api-provider-gcp/issues/903 - apiIntIPAddress, err := createInternalLB(ctx, in, masterSubnetSelflink, networkSelfLink, zones) - if err != nil { - return fmt.Errorf("failed to create internal load balancer address: %w", err) - } - // The firewall for masters, aka control-plane, is created by CAPG // Create the ones needed for worker to master communication if err = createFirewallRules(ctx, in, *gcpCluster.Status.Network.SelfLink); err != nil { @@ -214,6 +203,11 @@ func (p Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput) return fmt.Errorf("failed to create the private managed zone: %w", err) } + apiIntIPAddress, err := getInternalLBAddress(ctx, in.InstallConfig.Config.GCP.ProjectID, in.InstallConfig.Config.GCP.Region, getAPIAddressName(in.InfraID)) + if err != nil { + return fmt.Errorf("failed to get the internal load balancer address: %w", err) + } + // Create the public (optional) and private dns records if err := createDNSRecords(ctx, in.InstallConfig, in.InfraID, apiIPAddress, apiIntIPAddress); err != nil { return fmt.Errorf("failed to create DNS records: %w", err) diff --git a/pkg/infrastructure/gcp/clusterapi/network.go b/pkg/infrastructure/gcp/clusterapi/network.go index 22a786d8281..bcba0eb580e 100644 --- a/pkg/infrastructure/gcp/clusterapi/network.go +++ b/pkg/infrastructure/gcp/clusterapi/network.go @@ -3,21 +3,10 @@ package clusterapi import ( "context" "fmt" - "time" - - "github.com/sirupsen/logrus" - "google.golang.org/api/compute/v1" - - "github.com/openshift/installer/pkg/asset/manifests/gcp" - "github.com/openshift/installer/pkg/infrastructure/clusterapi" ) -func getAPIInternalResourceName(infraID string) string { - return fmt.Sprintf("%s-api-internal", infraID) -} - func getAPIAddressName(infraID string) string { - return fmt.Sprintf("%s-cluster-ip", infraID) + return fmt.Sprintf("%s-api-internal", infraID) } func getInternalLBAddress(ctx context.Context, project, region, name string) (string, error) { @@ -32,159 +21,3 @@ func getInternalLBAddress(ctx context.Context, project, region, name string) (st } return addrOutput.Address, nil } - -// createInternalLB creates a static ip address for the internal load balancer. -// Returns the IP address of the created load balancer. -func createInternalLB(ctx context.Context, in clusterapi.InfraReadyInput, subnetSelfLink, networkSelfLink string, zones []*string) (string, error) { - projectID := in.InstallConfig.Config.GCP.ProjectID - region := in.InstallConfig.Config.GCP.Region - name := getAPIAddressName(in.InfraID) - labels := mergeLabels(in.InstallConfig, in.InfraID) - - service, err := NewComputeService() - if err != nil { - return "", err - } - - ctx, cancel := context.WithTimeout(ctx, time.Minute*3) - defer cancel() - - // Patch the balancing mode on CAPG proxy classic load balancer backends - // to match the CONNECTION balancing mode used by installer-created - // internal passthrough LB, because: - // "all backend services that reference the instance group must use the same balancing mode" - // cf: https://cloud.google.com/load-balancing/docs/backend-service - logrus.Debug("Patching external load balancer") - extBesvcName := fmt.Sprintf("%s-apiserver", in.InfraID) - extBesvc, err := service.BackendServices.Get(projectID, extBesvcName).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("failed to get backend service: %w", err) - } - - for _, be := range extBesvc.Backends { - be.BalancingMode = "CONNECTION" - be.MaxConnections = int64(2 ^ 32) - } - - op, err := service.BackendServices.Patch(projectID, extBesvcName, extBesvc).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("failed to patch external load balancer: %w", err) - } - - if err := WaitForOperationGlobal(ctx, projectID, op); err != nil { - return "", fmt.Errorf("failed to wait for patching external load balancer: %w", err) - } - logrus.Debug("Successfully patched external load balancer") - - logrus.Debug("Creating internal load balancer") - addr := &compute.Address{ - Name: name, - AddressType: "INTERNAL", - Subnetwork: subnetSelfLink, - Description: resourceDescription, - Labels: labels, - Region: region, - } - - op, err = service.Addresses.Insert(projectID, region, addr).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("failed to create internal compute address: %w", err) - } - - if err := WaitForOperationRegional(ctx, projectID, region, op); err != nil { - return "", fmt.Errorf("failed to wait for compute address creation: %w", err) - } - - ipAddress, err := getInternalLBAddress(ctx, projectID, region, name) - if err != nil { - return "", fmt.Errorf("failed to get internal load balancer IP address: %w", err) - } - - hcName := getAPIInternalResourceName(in.InfraID) - healthCheck := &compute.HealthCheck{ - Region: region, - Name: hcName, - Description: resourceDescription, - HealthyThreshold: 3, - UnhealthyThreshold: 3, - CheckIntervalSec: 2, - TimeoutSec: 2, - Type: "HTTPS", - HttpsHealthCheck: &compute.HTTPSHealthCheck{ - Port: 6443, - RequestPath: "/readyz", - }, - } - - _, err = service.RegionHealthChecks.Insert(projectID, region, healthCheck).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("failed to create api-internal health check: %w", err) - } - - if err := WaitForOperationRegional(ctx, projectID, region, op); err != nil { - return "", fmt.Errorf("failed to wait for health check creation: %w", err) - } - - hc, err := service.RegionHealthChecks.Get(projectID, region, hcName).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("error getting health check: %w", err) - } - backends := []*compute.Backend{} - for _, zone := range zones { - igName := fmt.Sprintf("%s-%s-%s", in.InfraID, gcp.InstanceGroupRoleTag, *zone) - ig, err := service.InstanceGroups.Get(projectID, *zone, igName).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("error getting instance group %s in zone %s: %w", igName, *zone, err) - } - backends = append(backends, &compute.Backend{ - BalancingMode: "CONNECTION", - Group: ig.SelfLink, - }) - } - - besvcName := fmt.Sprintf("%s-api-internal", in.InfraID) - op, err = service.RegionBackendServices.Insert(projectID, region, &compute.BackendService{ - Backends: backends, - Name: besvcName, - LoadBalancingScheme: "INTERNAL", - Protocol: "TCP", - TimeoutSec: int64((10 * time.Minute).Seconds()), - HealthChecks: []string{hc.SelfLink}, - Region: region, - Network: networkSelfLink, - }).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("failed to create internal backend service: %w", err) - } - - if err := WaitForOperationRegional(ctx, projectID, region, op); err != nil { - return "", fmt.Errorf("failed to wait for internal backend service creation: %w", err) - } - - besvc, err := service.RegionBackendServices.Get(projectID, region, besvcName).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("failed to get backend service: %w", err) - } - - op, err = service.ForwardingRules.Insert(projectID, region, &compute.ForwardingRule{ - Name: fmt.Sprintf("%s-api-internal", in.InfraID), - IPProtocol: "TCP", - IPAddress: ipAddress, - LoadBalancingScheme: "INTERNAL", - Ports: []string{"6443", "22623"}, - BackendService: besvc.SelfLink, - Network: networkSelfLink, - Subnetwork: subnetSelfLink, - Region: region, - Labels: labels, - }).Context(ctx).Do() - if err != nil { - return "", fmt.Errorf("failed to create forwarding rule: %w", err) - } - - if err := WaitForOperationRegional(ctx, projectID, region, op); err != nil { - return "", fmt.Errorf("failed to wait for forwarding rule creation: %w", err) - } - logrus.Debug("Successfully created internal load balancer") - return ipAddress, nil -}