From 6a27c77ed2780adcabebfbed89d90afc3c42ab0c Mon Sep 17 00:00:00 2001 From: Bowei Du Date: Sun, 11 Aug 2019 08:31:14 -0700 Subject: [PATCH] Add IngressCRUD; adapter for extensions/v1beta1 and networking/v1beta1 IngressCRUD makes our e2e tests portable across older k8s API versions. e2e tests have been changed to use IngressCRUD. --- cmd/e2e-test/affinity_test.go | 14 +-- cmd/e2e-test/app_protocols_test.go | 33 ++++--- cmd/e2e-test/backend_config_test.go | 7 +- cmd/e2e-test/basic_https_test.go | 4 +- cmd/e2e-test/basic_test.go | 14 ++- cmd/e2e-test/cdn_test.go | 15 ++- cmd/e2e-test/draining_test.go | 15 +-- cmd/e2e-test/iap_test.go | 15 ++- cmd/e2e-test/security_policy_test.go | 30 +++--- cmd/e2e-test/timeout_test.go | 15 ++- cmd/e2e-test/upgrade_test.go | 9 +- pkg/e2e/fixtures.go | 14 ++- pkg/e2e/framework.go | 2 +- pkg/e2e/helpers.go | 18 ++-- pkg/e2e/legacy.go | 138 +++++++++++++++++++++++++++ pkg/e2e/legacy_test.go | 49 ++++++++++ pkg/e2e/sandbox.go | 1 + pkg/e2e/status.go | 3 +- 18 files changed, 300 insertions(+), 96 deletions(-) create mode 100644 pkg/e2e/legacy.go create mode 100644 pkg/e2e/legacy_test.go diff --git a/cmd/e2e-test/affinity_test.go b/cmd/e2e-test/affinity_test.go index 847f48c126..65aa8fe3fa 100644 --- a/cmd/e2e-test/affinity_test.go +++ b/cmd/e2e-test/affinity_test.go @@ -47,10 +47,6 @@ type affinityTransition struct { func TestAffinity(t *testing.T) { t.Parallel() - ing := fuzz.NewIngressBuilder("", "ingress-1", ""). - AddPath("test.com", "/", "service-1", intstr.FromInt(80)). - Build() - for _, tc := range []struct { desc string ttl int64 @@ -104,12 +100,16 @@ func TestAffinity(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + AddPath("test.com", "/", "service-1", intstr.FromInt(80)). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) - ing, err := e2e.WaitForIngress(s, ing, nil) + ing, err = e2e.WaitForIngress(s, ing, nil) if err != nil { t.Fatalf("error waiting for Ingress to stabilize: %v", err) } @@ -159,7 +159,7 @@ func TestAffinity(t *testing.T) { } // Wait for GCLB resources to be deleted. - if err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + if err := crud.Delete(s.Namespace, ing.Name); err != nil { t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) } diff --git a/cmd/e2e-test/app_protocols_test.go b/cmd/e2e-test/app_protocols_test.go index 3d34dd4cca..b982063bd1 100644 --- a/cmd/e2e-test/app_protocols_test.go +++ b/cmd/e2e-test/app_protocols_test.go @@ -20,7 +20,6 @@ import ( "context" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-gce/pkg/annotations" "k8s.io/ingress-gce/pkg/e2e" @@ -31,11 +30,6 @@ import ( func TestAppProtocol(t *testing.T) { t.Parallel() - ing := fuzz.NewIngressBuilder("", "ingress-1", ""). - DefaultBackend("service-1", intstr.FromString("https-port")). - AddPath("test.com", "/", "service-1", intstr.FromString("https-port")). - Build() - for _, tc := range []struct { desc string annotationVal string @@ -64,12 +58,17 @@ func TestAppProtocol(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + DefaultBackend("service-1", intstr.FromString("https-port")). + AddPath("test.com", "/", "service-1", intstr.FromString("https-port")). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) - ing, err := e2e.WaitForIngress(s, ing, nil) + ing, err = e2e.WaitForIngress(s, ing, nil) if err != nil { t.Fatalf("error waiting for Ingress to stabilize: %v", err) } @@ -83,7 +82,7 @@ func TestAppProtocol(t *testing.T) { } // Wait for GCLB resources to be deleted. - if err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + if err := crud.Delete(s.Namespace, ing.Name); err != nil { t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) } @@ -102,11 +101,6 @@ func TestAppProtocol(t *testing.T) { func TestAppProtocolTransition(t *testing.T) { t.Parallel() - ing := fuzz.NewIngressBuilder("", "ingress-1", ""). - DefaultBackend("service-1", intstr.FromString("https-port")). - AddPath("test.com", "/", "service-1", intstr.FromString("https-port")). - Build() - for _, tc := range []struct { desc string annotationVal string @@ -133,7 +127,12 @@ func TestAppProtocolTransition(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + DefaultBackend("service-1", intstr.FromString("https-port")). + AddPath("test.com", "/", "service-1", intstr.FromString("https-port")). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) @@ -154,7 +153,7 @@ func TestAppProtocolTransition(t *testing.T) { t.Fatalf("Error updating echo service: %v", err) } - ing, err := e2e.WaitForIngress(s, ing, nil) + ing, err = e2e.WaitForIngress(s, ing, nil) if err != nil { t.Fatalf("error waiting for Ingress to stabilize: %v", err) } @@ -167,7 +166,7 @@ func TestAppProtocolTransition(t *testing.T) { } // Wait for GCLB resources to be deleted. - if err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + if err := crud.Delete(s.Namespace, ing.Name); err != nil { t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) } diff --git a/cmd/e2e-test/backend_config_test.go b/cmd/e2e-test/backend_config_test.go index 4017018bd3..b69f80d1ad 100644 --- a/cmd/e2e-test/backend_config_test.go +++ b/cmd/e2e-test/backend_config_test.go @@ -101,10 +101,11 @@ func TestBackendConfigNegatives(t *testing.T) { } port80 := intstr.FromInt(80) - testIng := fuzz.NewIngressBuilder("", "ingress-1", ""). + testIng := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). AddPath("test.com", "/", "service-1", port80). Build() - testIng, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(testIng) + crud := e2e.IngressCRUD{C: Framework.Clientset} + testIng, err := crud.Create(testIng) if err != nil { t.Fatalf("error creating Ingress spec: %v", err) } @@ -133,7 +134,7 @@ func TestBackendConfigNegatives(t *testing.T) { t.Fatalf("error waiting for BackendConfig warning event: %v", err) } - testIng, err = Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Get(testIng.Name, metav1.GetOptions{}) + testIng, err = crud.Get(s.Namespace, testIng.Name) if err != nil { t.Fatalf("error retrieving Ingress %q: %v", testIng.Name, err) } diff --git a/cmd/e2e-test/basic_https_test.go b/cmd/e2e-test/basic_https_test.go index 907c6804c9..e1f0f07c3b 100644 --- a/cmd/e2e-test/basic_https_test.go +++ b/cmd/e2e-test/basic_https_test.go @@ -100,6 +100,7 @@ func TestBasicHTTPS(t *testing.T) { } } ing := tc.ingBuilder.Build() + ing.Namespace = s.Namespace // namespace depends on sandbox _, err := e2e.CreateEchoService(s, "service-1", nil) if err != nil { @@ -107,7 +108,8 @@ func TestBasicHTTPS(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) diff --git a/cmd/e2e-test/basic_test.go b/cmd/e2e-test/basic_test.go index 9f98c7adae..9f953b90a6 100644 --- a/cmd/e2e-test/basic_test.go +++ b/cmd/e2e-test/basic_test.go @@ -78,7 +78,9 @@ func TestBasic(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(tc.ing); err != nil { + crud := e2e.IngressCRUD{C: Framework.Clientset} + tc.ing.Namespace = s.Namespace // namespace depends on sandbox + if _, err = crud.Create(tc.ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, tc.ing.Name) @@ -132,11 +134,12 @@ func TestBasicStaticIP(t *testing.T) { } defer e2e.DeleteGCPAddress(s, addrName) - testIng := fuzz.NewIngressBuilder("", "ingress-1", ""). + testIng := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). DefaultBackend("service-1", intstr.FromInt(80)). AddStaticIP(addrName). Build() - testIng, err = Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(testIng) + crud := e2e.IngressCRUD{C: Framework.Clientset} + testIng, err = crud.Create(testIng) if err != nil { t.Fatalf("error creating Ingress spec: %v", err) } @@ -197,8 +200,9 @@ func TestEdge(t *testing.T) { t.Fatalf("error creating echo service: %v", err) } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(tc.ing); err != nil { + crud := e2e.IngressCRUD{C: Framework.Clientset} + tc.ing.Namespace = s.Namespace // namespace depends on sandbox + if _, err = crud.Create(tc.ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, tc.ing.Name) diff --git a/cmd/e2e-test/cdn_test.go b/cmd/e2e-test/cdn_test.go index 56e35cd36a..2f1d302052 100644 --- a/cmd/e2e-test/cdn_test.go +++ b/cmd/e2e-test/cdn_test.go @@ -22,7 +22,6 @@ import ( "reflect" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-gce/pkg/annotations" backendconfig "k8s.io/ingress-gce/pkg/apis/backendconfig/v1beta1" @@ -37,10 +36,6 @@ import ( func TestCDN(t *testing.T) { t.Parallel() - ing := fuzz.NewIngressBuilder("", "ingress-1", ""). - AddPath("test.com", "/", "service-1", intstr.FromInt(80)). - Build() - for _, tc := range []struct { desc string beConfig *backendconfig.BackendConfig @@ -90,12 +85,16 @@ func TestCDN(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + AddPath("test.com", "/", "service-1", intstr.FromInt(80)). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) - ing, err := e2e.WaitForIngress(s, ing, nil) + ing, err = e2e.WaitForIngress(s, ing, nil) if err != nil { t.Fatalf("error waiting for Ingress to stabilize: %v", err) } @@ -114,7 +113,7 @@ func TestCDN(t *testing.T) { } // Wait for GCLB resources to be deleted. - if err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + if err := crud.Delete(s.Namespace, ing.Name); err != nil { t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) } diff --git a/cmd/e2e-test/draining_test.go b/cmd/e2e-test/draining_test.go index ed3b00b6e7..f08a2dbef3 100644 --- a/cmd/e2e-test/draining_test.go +++ b/cmd/e2e-test/draining_test.go @@ -42,10 +42,6 @@ const ( func TestDraining(t *testing.T) { t.Parallel() - ing := fuzz.NewIngressBuilder("", "ingress-1", ""). - AddPath("test.com", "/", "service-1", intstr.FromInt(80)). - Build() - for _, tc := range []struct { desc string beConfig *backendconfig.BackendConfig @@ -86,12 +82,17 @@ func TestDraining(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + AddPath("test.com", "/", "service-1", intstr.FromInt(80)). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } + t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) - ing, err := e2e.WaitForIngress(s, ing, nil) + ing, err = e2e.WaitForIngress(s, ing, nil) if err != nil { t.Fatalf("error waiting for Ingress to stabilize: %v", err) } @@ -149,7 +150,7 @@ func TestDraining(t *testing.T) { deleteOptions := &fuzz.GCLBDeleteOptions{ SkipDefaultBackend: true, } - if err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + if err := crud.Delete(ing.Namespace, ing.Name); err != nil { t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) } t.Logf("Waiting for GCLB resources to be deleted (%s/%s)", s.Namespace, ing.Name) diff --git a/cmd/e2e-test/iap_test.go b/cmd/e2e-test/iap_test.go index 7dc7783a48..01179a894e 100644 --- a/cmd/e2e-test/iap_test.go +++ b/cmd/e2e-test/iap_test.go @@ -20,7 +20,6 @@ import ( "context" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-gce/pkg/annotations" backendconfig "k8s.io/ingress-gce/pkg/apis/backendconfig/v1beta1" @@ -36,10 +35,6 @@ import ( func TestIAP(t *testing.T) { t.Parallel() - ing := fuzz.NewIngressBuilder("", "ingress-1", ""). - AddPath("test.com", "/", "service-1", intstr.FromInt(80)). - Build() - for _, tc := range []struct { desc string beConfig *backendconfig.BackendConfig @@ -72,12 +67,16 @@ func TestIAP(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + AddPath("test.com", "/", "service-1", intstr.FromInt(80)). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) - ing, err := e2e.WaitForIngress(s, ing, nil) + ing, err = e2e.WaitForIngress(s, ing, nil) if err != nil { t.Fatalf("error waiting for Ingress to stabilize: %v", err) } @@ -91,7 +90,7 @@ func TestIAP(t *testing.T) { } // Wait for GCLB resources to be deleted. - if err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + if err := crud.Delete(ing.Namespace, ing.Name); err != nil { t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) } diff --git a/cmd/e2e-test/security_policy_test.go b/cmd/e2e-test/security_policy_test.go index 4fc0ddfe17..a0075c5775 100644 --- a/cmd/e2e-test/security_policy_test.go +++ b/cmd/e2e-test/security_policy_test.go @@ -56,7 +56,7 @@ func buildPolicyDisallowAll(name string) *computebeta.SecurityPolicy { return &computebeta.SecurityPolicy{ Name: name, Rules: []*computebeta.SecurityPolicyRule{ - &computebeta.SecurityPolicyRule{ + { Action: "deny(403)", Match: &computebeta.SecurityPolicyRuleMatcher{ Config: &computebeta.SecurityPolicyRuleMatcherConfig{ @@ -79,11 +79,11 @@ func TestSecurityPolicyEnable(t *testing.T) { buildPolicyAllowAll(fmt.Sprintf("enable-test-allow-all-%s", s.Namespace)), } defer func() { - if err := cleanupSecurityPolicies(t, ctx, Framework.Cloud, policies); err != nil { + if err := cleanupSecurityPolicies(ctx, t, Framework.Cloud, policies); err != nil { t.Errorf("cleanupSecurityPolicies(...) = %v, want nil", err) } }() - policies, err := createSecurityPolicies(t, ctx, Framework.Cloud, policies) + policies, err := createSecurityPolicies(ctx, t, Framework.Cloud, policies) if err != nil { t.Fatalf("createSecurityPolicies(...) = _, %v, want _, nil", err) } @@ -106,8 +106,12 @@ func TestSecurityPolicyEnable(t *testing.T) { t.Logf("Backend config %s/%s created", s.Namespace, testBackendConfig.Name) port80 := intstr.FromInt(80) - testIng := fuzz.NewIngressBuilder("", "ingress-1", "").DefaultBackend("service-1", port80).AddPath("test.com", "/", "service-1", port80).Build() - testIng, err = Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(testIng) + testIng := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + DefaultBackend("service-1", port80). + AddPath("test.com", "/", "service-1", port80). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + testIng, err = crud.Create(testIng) if err != nil { t.Fatalf("error creating Ingress spec: %v", err) } @@ -151,11 +155,11 @@ func TestSecurityPolicyTransition(t *testing.T) { buildPolicyDisallowAll(fmt.Sprintf("transition-test-disallow-all-%s", s.Namespace)), } defer func() { - if err := cleanupSecurityPolicies(t, ctx, Framework.Cloud, policies); err != nil { + if err := cleanupSecurityPolicies(ctx, t, Framework.Cloud, policies); err != nil { t.Errorf("cleanupSecurityPolicies(...) = %v, want nil", err) } }() - policies, err := createSecurityPolicies(t, ctx, Framework.Cloud, policies) + policies, err := createSecurityPolicies(ctx, t, Framework.Cloud, policies) if err != nil { t.Fatalf("createSecurityPolicies(...) = _, %v, want _, nil", err) } @@ -178,8 +182,12 @@ func TestSecurityPolicyTransition(t *testing.T) { t.Logf("Backend config %s/%s created", s.Namespace, testBackendConfig.Name) port80 := intstr.FromInt(80) - testIng := fuzz.NewIngressBuilder("", "ingress-1", "").DefaultBackend("service-1", port80).AddPath("test.com", "/", "service-1", port80).Build() - testIng, err = Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(testIng) + testIng := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + DefaultBackend("service-1", port80). + AddPath("test.com", "/", "service-1", port80). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + testIng, err = crud.Create(testIng) if err != nil { t.Fatalf("error creating Ingress spec: %v", err) } @@ -246,7 +254,7 @@ func TestSecurityPolicyTransition(t *testing.T) { }) } -func createSecurityPolicies(t *testing.T, ctx context.Context, c cloud.Cloud, policies []*computebeta.SecurityPolicy) ([]*computebeta.SecurityPolicy, error) { +func createSecurityPolicies(ctx context.Context, t *testing.T, c cloud.Cloud, policies []*computebeta.SecurityPolicy) ([]*computebeta.SecurityPolicy, error) { t.Logf("Creating security policies...") createdPolicies := []*computebeta.SecurityPolicy{} for _, policy := range policies { @@ -263,7 +271,7 @@ func createSecurityPolicies(t *testing.T, ctx context.Context, c cloud.Cloud, po return createdPolicies, nil } -func cleanupSecurityPolicies(t *testing.T, ctx context.Context, c cloud.Cloud, policies []*computebeta.SecurityPolicy) error { +func cleanupSecurityPolicies(ctx context.Context, t *testing.T, c cloud.Cloud, policies []*computebeta.SecurityPolicy) error { t.Logf("Deleting security policies...") var errs []string for _, policy := range policies { diff --git a/cmd/e2e-test/timeout_test.go b/cmd/e2e-test/timeout_test.go index fe7fd7994f..70db8eafa0 100644 --- a/cmd/e2e-test/timeout_test.go +++ b/cmd/e2e-test/timeout_test.go @@ -21,7 +21,6 @@ import ( "fmt" "testing" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/ingress-gce/pkg/annotations" backendconfig "k8s.io/ingress-gce/pkg/apis/backendconfig/v1beta1" @@ -36,10 +35,6 @@ const GLBDefaultTimeout int64 = 30 func TestTimeout(t *testing.T) { t.Parallel() - ing := fuzz.NewIngressBuilder("", "ingress-1", ""). - AddPath("test.com", "/", "service-1", intstr.FromInt(80)). - Build() - for _, tc := range []struct { desc string beConfig *backendconfig.BackendConfig @@ -77,12 +72,16 @@ func TestTimeout(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing); err != nil { + ing := fuzz.NewIngressBuilder(s.Namespace, "ingress-1", ""). + AddPath("test.com", "/", "service-1", intstr.FromInt(80)). + Build() + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, ing.Name) - ing, err := e2e.WaitForIngress(s, ing, nil) + ing, err = e2e.WaitForIngress(s, ing, nil) if err != nil { t.Fatalf("error waiting for Ingress to stabilize: %v", err) } @@ -110,7 +109,7 @@ func TestTimeout(t *testing.T) { deleteOptions := &fuzz.GCLBDeleteOptions{ SkipDefaultBackend: true, } - if err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + if err := crud.Delete(s.Namespace, ing.Name); err != nil { t.Errorf("Delete(%q) = %v, want nil", ing.Name, err) } t.Logf("Waiting for GCLB resources to be deleted (%s/%s)", s.Namespace, ing.Name) diff --git a/cmd/e2e-test/upgrade_test.go b/cmd/e2e-test/upgrade_test.go index e02a0b653e..2dadebd9ff 100644 --- a/cmd/e2e-test/upgrade_test.go +++ b/cmd/e2e-test/upgrade_test.go @@ -59,7 +59,9 @@ func TestUpgrade(t *testing.T) { } t.Logf("Echo service created (%s/%s)", s.Namespace, "service-1") - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(tc.ing); err != nil { + tc.ing.Namespace = s.Namespace // namespace depends on sandbox + crud := e2e.IngressCRUD{C: Framework.Clientset} + if _, err := crud.Create(tc.ing); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } t.Logf("Ingress created (%s/%s)", s.Namespace, tc.ing.Name) @@ -84,8 +86,9 @@ func TestUpgrade(t *testing.T) { newIng := fuzz.NewIngressBuilderFromExisting(tc.ing). AddPath("bar.com", "/", "service-1", port80). Build() - - if _, err := Framework.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Update(newIng); err != nil { + newIng.Namespace = s.Namespace // namespace depends on sandbox + // TODO: does the path need to be different for each upgrade + if _, err := crud.Update(newIng); err != nil { t.Fatalf("error creating Ingress spec: %v", err) } else { // If Ingress upgrade succeeds, we update the status on this Ingress diff --git a/pkg/e2e/fixtures.go b/pkg/e2e/fixtures.go index 5937dd9272..68a55aa65b 100644 --- a/pkg/e2e/fixtures.go +++ b/pkg/e2e/fixtures.go @@ -43,7 +43,7 @@ func CreateEchoService(s *Sandbox, name string, annotations map[string]string) ( return EnsureEchoService(s, name, annotations, v1.ServiceTypeNodePort, 1) } -// Ensures that the Echo service with the given description is set up +// EnsureEchoService that the Echo service with the given description is set up func EnsureEchoService(s *Sandbox, name string, annotations map[string]string, svcType v1.ServiceType, numReplicas int32) (*v1.Service, error) { expectedSvc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -157,18 +157,16 @@ func DeleteSecret(s *Sandbox, name string) error { // EnsureIngress creates a new Ingress or updates an existing one. func EnsureIngress(s *Sandbox, ing *v1beta1.Ingress) (*v1beta1.Ingress, error) { - currentIng, err := s.f.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Get(ing.ObjectMeta.Name, metav1.GetOptions{}) - + crud := &IngressCRUD{s.f.Clientset} + currentIng, err := crud.Get(ing.ObjectMeta.Namespace, ing.ObjectMeta.Name) if currentIng == nil || err != nil { - return s.f.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Create(ing) + return crud.Create(ing) } - // Update ingress spec if they are not equal if !reflect.DeepEqual(ing.Spec, currentIng.Spec) { - return s.f.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Update(ing) + return crud.Update(ing) } - - return currentIng, nil + return ing, nil } // NewGCPAddress reserves a global static IP address with the given name. diff --git a/pkg/e2e/framework.go b/pkg/e2e/framework.go index 8e4e75cade..d7c2492ca1 100644 --- a/pkg/e2e/framework.go +++ b/pkg/e2e/framework.go @@ -159,7 +159,7 @@ func (f *Framework) WithSandbox(testFunc func(*Sandbox) error) error { for _, s := range f.sandboxes { if s.Namespace == sandbox.Namespace { f.lock.Unlock() - return fmt.Errorf("sandbox %s was created previously by the framework.", s.Namespace) + return fmt.Errorf("sandbox %s was created previously by the framework", s.Namespace) } } klog.V(2).Infof("Using namespace %q for test sandbox", sandbox.Namespace) diff --git a/pkg/e2e/helpers.go b/pkg/e2e/helpers.go index 53d72e6df3..c109e048a3 100644 --- a/pkg/e2e/helpers.go +++ b/pkg/e2e/helpers.go @@ -55,7 +55,8 @@ type WaitForIngressOptions struct { func WaitForIngress(s *Sandbox, ing *v1beta1.Ingress, options *WaitForIngressOptions) (*v1beta1.Ingress, error) { err := wait.Poll(ingressPollInterval, ingressPollTimeout, func() (bool, error) { var err error - ing, err = s.f.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Get(ing.Name, metav1.GetOptions{}) + crud := IngressCRUD{s.f.Clientset} + ing, err = crud.Get(s.Namespace, ing.Name) if err != nil { return true, err } @@ -66,12 +67,11 @@ func WaitForIngress(s *Sandbox, ing *v1beta1.Ingress, options *WaitForIngressOpt result := validator.Check(context.Background()) if result.Err == nil { return true, nil - } else { - if options == nil || options.ExpectUnreachable { - return false, nil - } - return true, fmt.Errorf("unexpected error from validation: %v", result.Err) } + if options == nil || options.ExpectUnreachable { + return false, nil + } + return true, fmt.Errorf("unexpected error from validation: %v", result.Err) }) return ing, err } @@ -79,7 +79,8 @@ func WaitForIngress(s *Sandbox, ing *v1beta1.Ingress, options *WaitForIngressOpt // WaitForIngressDeletion deletes the given ingress and waits for the // resources associated with it to be deleted. func WaitForIngressDeletion(ctx context.Context, g *fuzz.GCLB, s *Sandbox, ing *v1beta1.Ingress, options *fuzz.GCLBDeleteOptions) error { - if err := s.f.Clientset.NetworkingV1beta1().Ingresses(s.Namespace).Delete(ing.Name, &metav1.DeleteOptions{}); err != nil { + crud := IngressCRUD{s.f.Clientset} + if err := crud.Delete(ing.Namespace, ing.Name); err != nil { return fmt.Errorf("delete(%q) = %v, want nil", ing.Name, err) } klog.Infof("Waiting for GCLB resources to be deleted (%s/%s), IngressDeletionOptions=%+v", s.Namespace, ing.Name, options) @@ -113,7 +114,7 @@ func WaitForNEGDeletion(ctx context.Context, c cloud.Cloud, g *fuzz.GCLB, option }) } -// WaitForNegConfiguration waits until the NEGStatus of the service is updated to at least one NEG +// WaitForNEGConfiguration waits until the NEGStatus of the service is updated to at least one NEG // TODO: (shance) make this more robust so it handles multiple NEGS func WaitForNEGConfiguration(svc *v1.Service, f *Framework, s *Sandbox) error { return wait.Poll(negPollInterval, gclbDeletionTimeout, func() (bool, error) { @@ -135,6 +136,7 @@ func WaitForNEGConfiguration(svc *v1.Service, f *Framework, s *Sandbox) error { }) } +// CheckGCLB whitebox testing is OK. func CheckGCLB(gclb *fuzz.GCLB, numForwardingRules int, numBackendServices int) error { // Do some cursory checks on the GCP objects. if len(gclb.ForwardingRule) != numForwardingRules { diff --git a/pkg/e2e/legacy.go b/pkg/e2e/legacy.go new file mode 100644 index 0000000000..47012b4f3b --- /dev/null +++ b/pkg/e2e/legacy.go @@ -0,0 +1,138 @@ +package e2e + +// Legacy conversion code for the old extensions v1beta1 API group. + +import ( + "bytes" + "encoding/json" + "errors" + + extv1beta1 "k8s.io/api/extensions/v1beta1" + "k8s.io/api/networking/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog" +) + +// IngressCRUD wraps basic CRUD to allow use of old and new APIs. +type IngressCRUD struct { + C *kubernetes.Clientset +} + +// Get Ingress resource. +func (crud *IngressCRUD) Get(ns, name string) (*v1beta1.Ingress, error) { + new, err := crud.supportsNewAPI() + if err != nil { + return nil, err + } + klog.Infof("Get %s/%s", ns, name) + if new { + return crud.C.NetworkingV1beta1().Ingresses(ns).Get(name, metav1.GetOptions{}) + } + klog.Warning("Using legacy API") + ing, err := crud.C.ExtensionsV1beta1().Ingresses(ns).Get(name, metav1.GetOptions{}) + return toIngressNetworkingGroup(ing), err +} + +// Create Ingress resource. +func (crud *IngressCRUD) Create(ing *v1beta1.Ingress) (*v1beta1.Ingress, error) { + new, err := crud.supportsNewAPI() + if err != nil { + return nil, err + } + klog.Infof("Create %s/%s", ing.Namespace, ing.Name) + if new { + return crud.C.NetworkingV1beta1().Ingresses(ing.Namespace).Create(ing) + } + klog.Warning("Using legacy API") + legacyIng := toIngressExtensionsGroup(ing) + legacyIng, err = crud.C.ExtensionsV1beta1().Ingresses(ing.Namespace).Create(legacyIng) + return toIngressNetworkingGroup(legacyIng), err +} + +// Update Ingress resource. +func (crud *IngressCRUD) Update(ing *v1beta1.Ingress) (*v1beta1.Ingress, error) { + new, err := crud.supportsNewAPI() + if err != nil { + return nil, err + } + klog.Infof("Update %s/%s", ing.Namespace, ing.Name) + if new { + return crud.C.NetworkingV1beta1().Ingresses(ing.Namespace).Update(ing) + } + klog.Warning("Using legacy API") + legacyIng := toIngressExtensionsGroup(ing) + legacyIng, err = crud.C.ExtensionsV1beta1().Ingresses(ing.Namespace).Update(legacyIng) + return toIngressNetworkingGroup(legacyIng), err +} + +// Delete Ingress resource. +func (crud *IngressCRUD) Delete(ns, name string) error { + new, err := crud.supportsNewAPI() + if err != nil { + return err + } + klog.Infof("Delete %s/%s", ns, name) + if new { + return crud.C.NetworkingV1beta1().Ingresses(ns).Delete(name, &metav1.DeleteOptions{}) + } + klog.Warning("Using legacy API") + return crud.C.ExtensionsV1beta1().Ingresses(ns).Delete(name, &metav1.DeleteOptions{}) +} + +func (crud *IngressCRUD) supportsNewAPI() (bool, error) { + if apiList, err := crud.C.Discovery().ServerResourcesForGroupVersion("networking/v1beta1"); err == nil { + for _, r := range apiList.APIResources { + if r.Kind == "Ingress" { + return true, nil + } + } + } + if apiList, err := crud.C.Discovery().ServerResourcesForGroupVersion("extensions/v1beta1"); err == nil { + for _, r := range apiList.APIResources { + if r.Kind == "Ingress" { + return false, nil + } + } + } + return false, errors.New("no Ingress resource found") +} + +func toIngressExtensionsGroup(ing *v1beta1.Ingress) *extv1beta1.Ingress { + b := &bytes.Buffer{} + e := json.NewEncoder(b) + // This is used by test cases from test data, so we assume no issues with + // conversion. + if err := e.Encode(ing); err != nil { + panic(err) + } + + ret := &extv1beta1.Ingress{} + d := json.NewDecoder(b) + if err := d.Decode(ret); err != nil { + panic(err) + } + + ret.APIVersion = "extensions/v1beta1" + + return ret +} + +func toIngressNetworkingGroup(ing *extv1beta1.Ingress) *v1beta1.Ingress { + b := &bytes.Buffer{} + e := json.NewEncoder(b) + // This is used by test cases from test data, so we assume no issues with + // conversion. + if err := e.Encode(ing); err != nil { + panic(err) + } + + ret := &v1beta1.Ingress{} + d := json.NewDecoder(b) + if err := d.Decode(ret); err != nil { + panic(err) + } + + ret.APIVersion = "networking.k8s.io/v1beta1" + return ret +} diff --git a/pkg/e2e/legacy_test.go b/pkg/e2e/legacy_test.go new file mode 100644 index 0000000000..d1030ea338 --- /dev/null +++ b/pkg/e2e/legacy_test.go @@ -0,0 +1,49 @@ +package e2e + +import ( + "reflect" + "testing" + + "github.com/kr/pretty" + extv1beta1 "k8s.io/api/extensions/v1beta1" + "k8s.io/api/networking/v1beta1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLegacyConversions(t *testing.T) { + for _, tc := range []struct { + ext *extv1beta1.Ingress + net *v1beta1.Ingress + }{ + { + ext: &extv1beta1.Ingress{ + TypeMeta: v1.TypeMeta{APIVersion: "extensions/v1beta1"}, + ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "foo"}, + Spec: extv1beta1.IngressSpec{ + Backend: &extv1beta1.IngressBackend{ + ServiceName: "svc1", + }, + }, + }, + net: &v1beta1.Ingress{ + TypeMeta: v1.TypeMeta{APIVersion: "networking.k8s.io/v1beta1"}, + ObjectMeta: v1.ObjectMeta{Namespace: "ns1", Name: "foo"}, + Spec: v1beta1.IngressSpec{ + Backend: &v1beta1.IngressBackend{ + ServiceName: "svc1", + }, + }, + }, + }, + } { + gotExt := toIngressExtensionsGroup(tc.net) + gotNet := toIngressNetworkingGroup(tc.ext) + + if !reflect.DeepEqual(gotExt, tc.ext) { + t.Errorf("Got\n%s\nwant\n%s", pretty.Sprint(gotExt), pretty.Sprint(tc.ext)) + } + if !reflect.DeepEqual(gotNet, tc.net) { + t.Errorf("Got%s\nwant\n%s", pretty.Sprint(gotNet), pretty.Sprint(tc.net)) + } + } +} diff --git a/pkg/e2e/sandbox.go b/pkg/e2e/sandbox.go index 00b9886e69..55dd3e89f4 100644 --- a/pkg/e2e/sandbox.go +++ b/pkg/e2e/sandbox.go @@ -80,6 +80,7 @@ func (s *Sandbox) Destroy() { s.destroyed = true } +// PutStatus into the status manager. func (s *Sandbox) PutStatus(status IngressStability) { s.f.statusManager.putStatus(s.Namespace, status) } diff --git a/pkg/e2e/status.go b/pkg/e2e/status.go index 2e8ae14629..187ae857a8 100644 --- a/pkg/e2e/status.go +++ b/pkg/e2e/status.go @@ -79,6 +79,7 @@ type StatusManager struct { informerRunning bool } +// NewStatusManager returns a new status manager. func NewStatusManager(f *Framework) *StatusManager { return &StatusManager{ cm: &v1.ConfigMap{ @@ -104,7 +105,7 @@ func (sm *StatusManager) init() error { } go func() { - for _ = range time.NewTicker(flushInterval).C { + for range time.NewTicker(flushInterval).C { sm.flush() } }()