From e0e60dbcdf0b0548892991273784f96325346d03 Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Fri, 28 Feb 2020 00:04:11 -0800 Subject: [PATCH] Add e2e test for configuring healthchecks via BackendConfig --- cmd/e2e-test/healthcheck_test.go | 175 +++++++++++++++++++++++++++++++ cmd/e2e-test/main_test.go | 3 + pkg/e2e/adapter/beconfig.go | 10 +- pkg/fuzz/gcp.go | 34 ++++++ pkg/fuzz/helpers.go | 12 ++- 5 files changed, 228 insertions(+), 6 deletions(-) create mode 100644 cmd/e2e-test/healthcheck_test.go diff --git a/cmd/e2e-test/healthcheck_test.go b/cmd/e2e-test/healthcheck_test.go new file mode 100644 index 0000000000..e51a17b4a7 --- /dev/null +++ b/cmd/e2e-test/healthcheck_test.go @@ -0,0 +1,175 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "testing" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-gce/pkg/annotations" + backendconfig "k8s.io/ingress-gce/pkg/apis/backendconfig/v1" + "k8s.io/ingress-gce/pkg/e2e" + "k8s.io/ingress-gce/pkg/e2e/adapter" + "k8s.io/ingress-gce/pkg/fuzz" + "k8s.io/ingress-gce/pkg/fuzz/features" +) + +func TestHealthCheck(t *testing.T) { + t.Parallel() + + pint64 := func(x int64) *int64 { return &x } + pstring := func(x string) *string { return &x } + + for _, tc := range []struct { + desc string + beConfig *backendconfig.BackendConfig + want *backendconfig.HealthCheckConfig + }{ + { + desc: "override healthcheck path", + beConfig: fuzz.NewBackendConfigBuilder("", "backendconfig-1").Build(), + want: &backendconfig.HealthCheckConfig{ + CheckIntervalSec: pint64(7), + TimeoutSec: pint64(3), + HealthyThreshold: pint64(3), + UnhealthyThreshold: pint64(5), + RequestPath: pstring("/my-path"), + }, + }, + } { + tc := tc // Capture tc as we are running this in parallel. + Framework.RunWithSandbox(tc.desc, t, func(t *testing.T, s *e2e.Sandbox) { + t.Parallel() + + ctx := context.Background() + + backendConfigAnnotation := map[string]string{ + annotations.BackendConfigKey: `{"default":"backendconfig-1"}`, + } + tc.beConfig.Spec.HealthCheck = tc.want + + if _, err := Framework.BackendConfigClient.CloudV1().BackendConfigs(s.Namespace).Create(tc.beConfig); err != nil { + t.Fatalf("error creating BackendConfig: %v", err) + } + t.Logf("BackendConfig created (%s/%s) ", s.Namespace, tc.beConfig.Name) + + _, err := e2e.CreateEchoService(s, "service-1", backendConfigAnnotation) + if err != nil { + t.Fatalf("error creating echo service: %v", err) + } + t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") + + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + DefaultBackend("service-1", intstr.FromInt(80)). + Build() + crud := adapter.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { + t.Fatalf("error creating Ingress spec: %v", err) + } + t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) + + ing, err = e2e.WaitForIngress(s, ing, nil) + if err != nil { + t.Fatalf("error waiting for Ingress to stabilize: %v", err) + } + t.Logf("GCLB resources created (%s/%s)", s.Namespace, ing.Name) + + vip := ing.Status.LoadBalancer.Ingress[0].IP + t.Logf("Ingress %s/%s VIP = %s", s.Namespace, ing.Name, vip) + params := &fuzz.GCLBForVIPParams{VIP: vip, Validators: fuzz.FeatureValidators(features.All)} + gclb, err := fuzz.GCLBForVIP(context.Background(), Framework.Cloud, params) + if err != nil { + t.Fatalf("Error getting GCP resources for LB with IP = %q: %v", vip, err) + } + + verifyHealthCheck(t, gclb, tc.want) + + // Wait for GCLB resources to be deleted. + if err := crud.Delete(s.Namespace, ing.Name); err != nil { + t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) + } + + deleteOptions := &fuzz.GCLBDeleteOptions{ + SkipDefaultBackend: true, + } + t.Logf("Waiting for GCLB resources to be deleted (%s/%s)", s.Namespace, ing.Name) + if err := e2e.WaitForGCLBDeletion(ctx, Framework.Cloud, gclb, deleteOptions); err != nil { + t.Errorf("e2e.WaitForGCLBDeletion(...) = %v, want nil", err) + } + t.Logf("GCLB resources deleted (%s/%s)", s.Namespace, ing.Name) + }) + } +} + +func verifyHealthCheck(t *testing.T, gclb *fuzz.GCLB, want *backendconfig.HealthCheckConfig) { + // We assume there is a single service for now. The logic will have to be + // changed if there is more than one backend service. + for _, bs := range gclb.BackendService { + for _, hcURL := range bs.GA.HealthChecks { + rID, err := cloud.ParseResourceURL(hcURL) + if err != nil { + t.Fatalf("cloud.ParseResourceURL(%q) = _, %v; want _, nil", hcURL, err) + } + hc, ok := gclb.HealthCheck[*rID.Key] + if !ok { + t.Fatalf("HealthCheck %s not found in BackendService %+v", rID.Key, bs) + } + + // Pull out the field that are in common among the different + // healthchecks per protocol. + common := struct { + port int64 + requestPath string + }{} + switch { + case hc.GA.Http2HealthCheck != nil: + common.port = hc.GA.Http2HealthCheck.Port + common.requestPath = hc.GA.Http2HealthCheck.RequestPath + case hc.GA.HttpHealthCheck != nil: + common.port = hc.GA.HttpHealthCheck.Port + common.requestPath = hc.GA.HttpHealthCheck.RequestPath + case hc.GA.HttpsHealthCheck != nil: + common.port = hc.GA.HttpsHealthCheck.Port + common.requestPath = hc.GA.HttpsHealthCheck.RequestPath + } + + if want.CheckIntervalSec != nil && hc.GA.CheckIntervalSec != *want.CheckIntervalSec { + t.Errorf("HealthCheck %v checkIntervalSec = %d, want %d", rID.Key, hc.GA.CheckIntervalSec, *want.CheckIntervalSec) + } + if want.TimeoutSec != nil && hc.GA.TimeoutSec != *want.TimeoutSec { + t.Errorf("HealthCheck %v timeoutSec = %d, want %d", rID.Key, hc.GA.TimeoutSec, *want.TimeoutSec) + } + if want.HealthyThreshold != nil && hc.GA.HealthyThreshold != *want.HealthyThreshold { + t.Errorf("HealthCheck %v healthyThreshold = %d, want %d", rID.Key, hc.GA.HealthyThreshold, *want.HealthyThreshold) + } + if want.UnhealthyThreshold != nil && hc.GA.UnhealthyThreshold != *want.UnhealthyThreshold { + t.Errorf("HealthCheck %v unhealthThreshold = %d, want %d", rID.Key, hc.GA.UnhealthyThreshold, *want.UnhealthyThreshold) + } + if want.Type != nil { + t.Errorf("HealthCheck %v type = %s, want %s", rID.Key, hc.GA.Type, *want.Type) + } + if want.Port != nil && common.port != *want.Port { + t.Errorf("HealthCheck %v port = %d, want %d", rID.Key, common.port, *want.Port) + } + if want.RequestPath != nil && common.requestPath != *want.RequestPath { + t.Errorf("HealthCheck %v requestPath = %q, want %q", rID.Key, common.requestPath, *want.RequestPath) + } + } + } +} diff --git a/cmd/e2e-test/main_test.go b/cmd/e2e-test/main_test.go index 92e217c102..0df6ccf8a4 100644 --- a/cmd/e2e-test/main_test.go +++ b/cmd/e2e-test/main_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "github.com/kr/pretty" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" "k8s.io/ingress-gce/pkg/e2e" @@ -78,6 +79,8 @@ func init() { func TestMain(m *testing.M) { flag.Parse() + fmt.Fprintf(os.Stderr, "Flags:\n%s\n", pretty.Sprint(flags)) + if !flags.inCluster && !flags.run { fmt.Fprintln(os.Stderr, "Set -run to run the tests.") // Return 0 here so 'go test ./...' will succeed. diff --git a/pkg/e2e/adapter/beconfig.go b/pkg/e2e/adapter/beconfig.go index 0a8a79c210..eb94fcab7a 100644 --- a/pkg/e2e/adapter/beconfig.go +++ b/pkg/e2e/adapter/beconfig.go @@ -22,7 +22,7 @@ import ( "errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/ingress-gce/pkg/apis/backendconfig/v1" + v1 "k8s.io/ingress-gce/pkg/apis/backendconfig/v1" "k8s.io/ingress-gce/pkg/apis/backendconfig/v1beta1" client "k8s.io/ingress-gce/pkg/backendconfig/client/clientset/versioned" "k8s.io/klog" @@ -39,11 +39,11 @@ func (crud *BackendConfigCRUD) Get(ns, name string) (*v1.BackendConfig, error) { if err != nil { return nil, err } - klog.V(2).Infof("Get BackendConfig %s/%s", ns, name) + klog.V(3).Infof("Get BackendConfig %s/%s", ns, name) if isV1 { return crud.C.CloudV1().BackendConfigs(ns).Get(name, metav1.GetOptions{}) } - klog.V(2).Info("Using BackendConfig V1beta1 API") + klog.V(4).Info("Using BackendConfig V1beta1 API") bc, err := crud.C.CloudV1beta1().BackendConfigs(ns).Get(name, metav1.GetOptions{}) return toV1(bc), err } @@ -100,11 +100,11 @@ func (crud *BackendConfigCRUD) List(ns string) (*v1.BackendConfigList, error) { if err != nil { return nil, err } - klog.V(2).Infof("List BackendConfigs in namespace(%s)", ns) + klog.V(3).Infof("List BackendConfigs in namespace(%s)", ns) if isV1 { return crud.C.CloudV1().BackendConfigs(ns).List(metav1.ListOptions{}) } - klog.V(2).Info("Using BackendConfig V1beta1 API") + klog.V(4).Info("Using BackendConfig V1beta1 API") bcl, err := crud.C.CloudV1beta1().BackendConfigs(ns).List(metav1.ListOptions{}) return toV1List(bcl), err } diff --git a/pkg/fuzz/gcp.go b/pkg/fuzz/gcp.go index 1c654b64f8..ec20486266 100644 --- a/pkg/fuzz/gcp.go +++ b/pkg/fuzz/gcp.go @@ -85,6 +85,11 @@ type BackendService struct { Beta *computebeta.BackendService } +// HealthCheck is a union of the API version types. +type HealthCheck struct { + GA *compute.HealthCheck +} + // NetworkEndpointGroup is a union of the API version types. type NetworkEndpointGroup struct { GA *compute.NetworkEndpointGroup @@ -114,6 +119,7 @@ type GCLB struct { BackendService map[meta.Key]*BackendService NetworkEndpointGroup map[meta.Key]*NetworkEndpointGroup InstanceGroup map[meta.Key]*InstanceGroup + HealthCheck map[meta.Key]*HealthCheck } // NewGCLB returns an empty GCLB. @@ -127,6 +133,7 @@ func NewGCLB(vip string) *GCLB { BackendService: map[meta.Key]*BackendService{}, NetworkEndpointGroup: map[meta.Key]*NetworkEndpointGroup{}, InstanceGroup: map[meta.Key]*InstanceGroup{}, + HealthCheck: map[meta.Key]*HealthCheck{}, } } @@ -559,6 +566,20 @@ func GCLBForVIP(ctx context.Context, c cloud.Cloud, params *GCLBForVIPParams) (* } gclb.BackendService[*bsKey].Beta = bs } + + for _, hcURL := range bs.HealthChecks { + rID, err := cloud.ParseResourceURL(hcURL) + if err != nil { + return nil, err + } + hc, err := c.HealthChecks().Get(ctx, rID.Key) + if err != nil { + return nil, err + } + gclb.HealthCheck[*rID.Key] = &HealthCheck{ + GA: hc, + } + } } var negKeys []*meta.Key @@ -799,6 +820,19 @@ func RegionalGCLBForVIP(ctx context.Context, c cloud.Cloud, gclb *GCLB, params * } gclb.BackendService[*bsKey].Beta = bs } + for _, hcURL := range bs.HealthChecks { + rID, err := cloud.ParseResourceURL(hcURL) + if err != nil { + return err + } + hc, err := c.RegionHealthChecks().Get(ctx, rID.Key) + if err != nil { + return err + } + gclb.HealthCheck[*rID.Key] = &HealthCheck{ + GA: hc, + } + } } var negKeys []*meta.Key diff --git a/pkg/fuzz/helpers.go b/pkg/fuzz/helpers.go index 6f423c138f..1171b5b623 100644 --- a/pkg/fuzz/helpers.go +++ b/pkg/fuzz/helpers.go @@ -21,7 +21,7 @@ import ( "strconv" "strings" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/api/networking/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -394,6 +394,7 @@ func (b *BackendConfigBuilder) SetConnectionDrainingTimeout(timeout int64) *Back return b } +// AddCustomRequestHeader adds a custom request header to the BackendConfig. func (b *BackendConfigBuilder) AddCustomRequestHeader(header string) *BackendConfigBuilder { if b.backendConfig.Spec.CustomRequestHeaders == nil { b.backendConfig.Spec.CustomRequestHeaders = &backendconfig.CustomRequestHeadersConfig{} @@ -401,3 +402,12 @@ func (b *BackendConfigBuilder) AddCustomRequestHeader(header string) *BackendCon b.backendConfig.Spec.CustomRequestHeaders.Headers = append(b.backendConfig.Spec.CustomRequestHeaders.Headers, header) return b } + +// SetHealthCheckPath adds a health check path override. +func (b *BackendConfigBuilder) SetHealthCheckPath(path string) *BackendConfigBuilder { + if b.backendConfig.Spec.HealthCheck == nil { + b.backendConfig.Spec.HealthCheck = &backendconfig.HealthCheckConfig{} + } + b.backendConfig.Spec.HealthCheck.RequestPath = &path + return b +}