diff --git a/terratest/test/k8gb_basic_app_test.go b/terratest/test/k8gb_basic_app_test.go index 8b682df2a1..dadc80d8a5 100644 --- a/terratest/test/k8gb_basic_app_test.go +++ b/terratest/test/k8gb_basic_app_test.go @@ -67,7 +67,7 @@ func TestK8gbBasicAppExample(t *testing.T) { utils.CreateGslb(t, options, settings, kubeResourcePath) - k8s.WaitUntilIngressAvailable(t, options, "test-gslb", 120, 1*time.Second) + k8s.WaitUntilIngressAvailable(t, options, "test-gslb", utils.DefaultRetries, 1*time.Second) ingress := k8s.GetIngress(t, options, "test-gslb") require.Equal(t, ingress.Name, "test-gslb") diff --git a/terratest/test/k8gb_lifecycle_test.go b/terratest/test/k8gb_lifecycle_test.go index 36f91fa2e7..913c4f34b4 100644 --- a/terratest/test/k8gb_lifecycle_test.go +++ b/terratest/test/k8gb_lifecycle_test.go @@ -71,7 +71,7 @@ func TestK8gbRepeatedlyRecreatedFromIngress(t *testing.T) { utils.CreateGslb(t, options, settings, ingressResourcePath) - k8s.WaitUntilIngressAvailable(t, options, name, 120, 1*time.Second) + k8s.WaitUntilIngressAvailable(t, options, name, utils.DefaultRetries, 1*time.Second) ingress := k8s.GetIngress(t, options, name) @@ -87,7 +87,7 @@ func TestK8gbRepeatedlyRecreatedFromIngress(t *testing.T) { // recreate ingress utils.CreateGslb(t, options, settings, ingressResourcePath) - k8s.WaitUntilIngressAvailable(t, options, name, 120, 1*time.Second) + k8s.WaitUntilIngressAvailable(t, options, name, utils.DefaultRetries, 1*time.Second) ingress = k8s.GetIngress(t, options, name) @@ -129,14 +129,14 @@ func TestK8gbSpecKeepsStableAfterIngressUpdates(t *testing.T) { // create gslb utils.CreateGslb(t, options, settings, kubeResourcePath) - k8s.WaitUntilIngressAvailable(t, options, name, 120, 1*time.Second) + k8s.WaitUntilIngressAvailable(t, options, name, utils.DefaultRetries, 1*time.Second) assertStrategy(t, options) // reapply ingress utils.CreateGslb(t, options, settings, ingressResourcePath) - k8s.WaitUntilIngressAvailable(t, options, name, 120, 1*time.Second) + k8s.WaitUntilIngressAvailable(t, options, name, utils.DefaultRetries, 1*time.Second) ingress := k8s.GetIngress(t, options, name) diff --git a/terratest/test/k8gb_weight_test.go b/terratest/test/k8gb_weight_test.go index b91a63781b..1c14179509 100644 --- a/terratest/test/k8gb_weight_test.go +++ b/terratest/test/k8gb_weight_test.go @@ -29,7 +29,8 @@ import ( func TestWeightsExistsInLocalDNSEndpoint(t *testing.T) { t.Parallel() const host = "terratest-roundrobin.cloud.example.com" - const localtargets = "localtargets-" + host + const endpointDNSNameEU = "gslb-ns-eu-cloud.example.com" + const endpointDNSNameUS = "gslb-ns-us-cloud.example.com" const gslbPath = "../examples/roundrobin_weight1.yaml" instanceEU, err := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). WithGslb(gslbPath, host). @@ -50,31 +51,46 @@ func TestWeightsExistsInLocalDNSEndpoint(t *testing.T) { err = instanceUS.WaitForAppIsRunning() require.NoError(t, err) - epLocalEU,err := instanceEU.GetExternalDNSEndpoint().GetEndpointByName(localtargets) - require.NoError(t, err, "missing EU endpoint", localtargets) - epLocalUS,err := instanceUS.GetExternalDNSEndpoint().GetEndpointByName(localtargets) - require.NoError(t, err, "missing US endpoint", localtargets) - expectedTargets := append(epLocalEU.Targets,epLocalUS.Targets...) + err = instanceEU.WaitForExternalDNSEndpointExists() + require.NoError(t, err) + err = instanceUS.WaitForExternalDNSEndpointExists() + require.NoError(t, err) + + err = instanceEU.WaitForLocalDNSEndpointExists() + require.NoError(t, err) + err = instanceUS.WaitForLocalDNSEndpointExists() + require.NoError(t, err) - err = instanceEU.WaitForExpected(expectedTargets) + err = instanceEU.Resources().WaitForExternalDNSEndpointHasTargets(endpointDNSNameEU) require.NoError(t, err) - err = instanceUS.WaitForExpected(expectedTargets) + err = instanceUS.Resources().WaitForExternalDNSEndpointHasTargets(endpointDNSNameUS) require.NoError(t, err) + epExternalEU, err := instanceEU.Resources().GetExternalDNSEndpoint().GetEndpointByName(endpointDNSNameEU) + require.NoError(t, err, "missing EU endpoint %s", endpointDNSNameEU) + epExternalUS, err := instanceUS.Resources().GetExternalDNSEndpoint().GetEndpointByName(endpointDNSNameUS) + require.NoError(t, err, "missing US endpoint %s", endpointDNSNameUS) + expectedTargets := append(epExternalEU.Targets, epExternalUS.Targets...) + + err = instanceUS.WaitForLocalDNSEndpointHasTargets(expectedTargets) + require.NoError(t, err, "US expectedTargets %v", expectedTargets) + err = instanceEU.WaitForLocalDNSEndpointHasTargets(expectedTargets) + require.NoError(t, err, "EU expectedTargets %v", expectedTargets) + for _, instance := range []*utils.Instance{instanceEU, instanceUS} { - ep, err := instance.GetExternalDNSEndpoint().GetEndpointByName(host) + ep, err := instance.Resources().GetLocalDNSEndpoint().GetEndpointByName(host) require.NoError(t, err, "missing endpoint", host) // check all labels are correct require.Equal(t, "roundRobin", ep.Labels["strategy"]) - require.True(t, Contains(ep.Labels["weight-eu-0-50"], epLocalEU.Targets)) - require.True(t, Contains(ep.Labels["weight-eu-1-50"], epLocalEU.Targets)) - require.True(t, Contains(ep.Labels["weight-us-0-50"], epLocalUS.Targets)) - require.True(t, Contains(ep.Labels["weight-us-1-50"], epLocalUS.Targets)) require.NotEqual(t, ep.Labels["weight-eu-0-50"], ep.Labels["weight-eu-1-50"]) require.NotEqual(t, ep.Labels["weight-us-0-50"], ep.Labels["weight-us-1-50"]) // check all targets are correct - for _, v := range epLocalEU.Targets { require.True(t, Contains(v, ep.Targets)) } - for _, v := range epLocalUS.Targets { require.True(t, Contains(v, ep.Targets)) } + for _, v := range epExternalEU.Targets { + require.True(t, Contains(v, ep.Targets)) + } + for _, v := range epExternalEU.Targets { + require.True(t, Contains(v, ep.Targets)) + } } } diff --git a/terratest/utils/extensions.go b/terratest/utils/extensions.go index c6c9f354b2..a6a7e35a79 100644 --- a/terratest/utils/extensions.go +++ b/terratest/utils/extensions.go @@ -38,6 +38,7 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -127,6 +128,8 @@ func (w *Workflow) WithIngress(path string) *Workflow { return w } +// WithGslb +// TODO: consider taking host dynamically func (w *Workflow) WithGslb(path, host string) *Workflow { var err error if host == "" { @@ -195,13 +198,13 @@ func (w *Workflow) Start() (*Instance, error) { testAppFilter := metav1.ListOptions{ LabelSelector: "app.kubernetes.io/name=" + w.state.testApp.name, } - k8s.WaitUntilNumPodsCreated(w.t, w.k8sOptions, testAppFilter, 1, 120, 1*time.Second) + k8s.WaitUntilNumPodsCreated(w.t, w.k8sOptions, testAppFilter, 1, DefaultRetries, 1*time.Second) var testAppPods []corev1.Pod testAppPods = k8s.ListPods(w.t, w.k8sOptions, testAppFilter) for _, pod := range testAppPods { - k8s.WaitUntilPodAvailable(w.t, w.k8sOptions, pod.Name, 120, 1*time.Second) + k8s.WaitUntilPodAvailable(w.t, w.k8sOptions, pod.Name, DefaultRetries, 1*time.Second) } - k8s.WaitUntilServiceAvailable(w.t, w.k8sOptions, w.state.testApp.name, 120, 1*time.Second) + k8s.WaitUntilServiceAvailable(w.t, w.k8sOptions, w.state.testApp.name, DefaultRetries, 1*time.Second) w.state.testApp.isRunning = true } @@ -245,6 +248,21 @@ func (i *Instance) Kill() { } } +func (i *Instance) ReapplyIngress(path string) { + var err error + i.w.t.Logf("reapplying %s", path) + i.w.settings.ingressResourcePath = path + i.w.settings.gslbResourcePath = "" + i.w.state.gslb.name, err = i.w.getManifestName(i.w.settings.ingressResourcePath) + require.NoError(i.w.t, err) + k8s.KubectlApply(i.w.t, i.w.k8sOptions, i.w.settings.ingressResourcePath) + // modifying inner state.gslb.name and ingress.Name has nothing to do with reading these values dynamically afterwards. + k8s.WaitUntilIngressAvailable(i.w.t, i.w.k8sOptions, i.w.state.gslb.name, 60, 1*time.Second) + ingress := k8s.GetIngress(i.w.t, i.w.k8sOptions, i.w.state.gslb.name) + require.Equal(i.w.t, ingress.Name, i.w.state.gslb.name) + i.w.settings.ingressName = i.w.state.gslb.name +} + // GetCoreDNSIP gets core DNS IP address func (i *Instance) GetCoreDNSIP() string { cmd := shell.Command{ @@ -297,7 +315,48 @@ func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) { return waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs, i.w.settings.digUsingUDP) } -// WaitForExpected waits until GSLB dig doesnt return list of expected IP's +func (i *Instance) WaitForLocalDNSEndpointExists() error { + periodic := func() (result bool, err error) { + lep := i.Resources().GetLocalDNSEndpoint() + result = len(lep.Spec.Endpoints) > 0 + return result, err + } + return tickerWaiter(DefaultRetries, "LocalDNSEndpoint exists:", periodic) +} + +func (i *Instance) WaitForExternalDNSEndpointExists() error { + periodic := func() (result bool, err error) { + lep := i.Resources().GetExternalDNSEndpoint() + result = len(lep.Spec.Endpoints) > 0 + return result, err + } + return tickerWaiter(DefaultRetries, "ExternalDNSEndpoint exists:", periodic) +} + +func (r *Resources) WaitForExternalDNSEndpointHasTargets(epName string) error { + periodic := func() (result bool, err error) { + epx, err := r.GetExternalDNSEndpoint().GetEndpointByName(epName) + if err != nil { + return false, nil + } + result = len(epx.Targets) > 0 + return result, err + } + return tickerWaiter(DefaultRetries, "ExternalDNSEndpoint has targets:", periodic) +} + +// WaitForLocalDNSEndpointHasTargets waits until LocalDNSEndpoint has expected targets +func (i *Instance) WaitForLocalDNSEndpointHasTargets(expectedIPs []string) error { + periodic := func() (result bool, err error) { + lep := i.Resources().GetLocalDNSEndpoint() + ep, err := lep.GetEndpointByName(i.w.state.gslb.host) + result = EqualStringSlices(ep.Targets, expectedIPs) + return result, err + } + return tickerWaiter(DefaultRetries, "LocalDNSEndpoint targets:", periodic) +} + +// WaitForExpected waits until GSLB dig doesn't return list of expected IP's func (i *Instance) WaitForExpected(expectedIPs []string) (err error) { _, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs, i.w.settings.digUsingUDP) if err != nil { @@ -320,7 +379,7 @@ func (i *Instance) WaitForAppIsStopped() (err error) { func (i *Instance) waitForApp(predicate func(instances int) bool) (err error) { const ( description = "Wait for app is running" - maxRetries = 50 + maxRetries = 50 waitSeconds = 2 ) @@ -361,22 +420,10 @@ func (i *Instance) Dig() []string { func (i *Instance) GetLocalTargets() []string { dnsName := fmt.Sprintf("localtargets-%s", i.w.state.gslb.host) dig, err := dns.Dig("localhost:"+strconv.Itoa(i.w.state.gslb.port), dnsName, i.w.settings.digUsingUDP) - require.NoError(i.w.t, err) + i.logIfError(err, "GetLocalTargets(), dig: %s", err) return dig } -func (i *Instance) GetExternalDNSEndpoint() (ep DNSEndpoint) { - const endpointName = "test-gslb" - ep = DNSEndpoint{} - j, err := k8s.RunKubectlAndGetOutputE(i.w.t, i.w.k8sOptions, "get", "dnsendpoints.externaldns.k8s.io", endpointName, "-ojson") - require.NoError(i.w.t, err) - err = json.Unmarshal([]byte(j), &ep) - if err != nil { - return ep - } - return ep -} - // HitTestApp makes HTTP GET to TestApp when installed otherwise panics. // If the function successfully hits the TestApp, it returns the TestAppResult. func (i *Instance) HitTestApp() (result *TestAppResult) { @@ -461,8 +508,86 @@ func waitForLocalGSLBNew(t *testing.T, host string, port int, expectedResult []s return DoWithRetryWaitingForValueE( t, "Wait for failover to happen and coredns to pickup new values...", - 120, + DefaultRetries, time.Second*1, func() ([]string, error) { return dns.Dig("localhost:"+strconv.Itoa(port), host, isUdp) }, expectedResult) } + +// tickerWaiter periodically executes periodic function +func tickerWaiter(timeoutSeconds int, name string, periodic func() (result bool, err error)) error { + const interval = 2 + var cycles = timeoutSeconds / interval + t := 0 + for range time.NewTicker(interval * time.Second).C { + result, err := periodic() + if err != nil { + return fmt.Errorf("%s: targets %v, error: %s", name, result, err) + } + if result { + return nil + } + if t >= cycles { + break + } + t++ + } + return fmt.Errorf("%s: timeout", name) +} + +func (i *Instance) Resources() (o *Resources) { + return &Resources{ + i, + } +} + +func (i *Instance) continueIfK8sResourceNotFound(err error) { + if err != nil && strings.HasSuffix(err.Error(), "not found") { + return + } + require.NoError(i.w.t, err) +} + +type Resources struct { + i *Instance +} + +// GslbSpecProperty returns actual value of one Spec property, e.g: `spec.ingress.rules[0].host` +func (r *Resources) GslbSpecProperty(specPath string) string { + actualValue, _ := k8s.RunKubectlAndGetOutputE(r.i.w.t, r.i.w.k8sOptions, "get", "gslb", r.i.w.state.gslb.name, + "-o", fmt.Sprintf("custom-columns=SERVICESTATUS:%s", specPath), "--no-headers") + return actualValue +} + +func (r *Resources) Ingress() *networkingv1.Ingress { + return k8s.GetIngress(r.i.w.t, r.i.w.k8sOptions, r.i.w.settings.ingressName) +} + +func (r *Resources) GetLocalDNSEndpoint() DNSEndpoint { + ep, err := r.getDNSEndpoint("test-gslb", r.i.w.namespace) + r.i.continueIfK8sResourceNotFound(err) + return ep +} + +func (r *Resources) GetExternalDNSEndpoint() DNSEndpoint { + ep, err := r.getDNSEndpoint("k8gb-ns-extdns", "k8gb") + r.i.continueIfK8sResourceNotFound(err) + return ep +} + +func (i *Instance) logIfError(err error, message string, args ...interface{}) { + if err != nil { + i.w.t.Logf(message, args) + } +} + +func (r *Resources) getDNSEndpoint(epName, ns string) (ep DNSEndpoint, err error) { + ep = DNSEndpoint{} + opts := k8s.NewKubectlOptions(r.i.w.k8sOptions.ContextName, r.i.w.k8sOptions.ConfigPath, ns) + j, err := k8s.RunKubectlAndGetOutputE(r.i.w.t, opts, "get", "dnsendpoints.externaldns.k8s.io", epName, "-ojson") + if err != nil { + return ep, err + } + err = json.Unmarshal([]byte(j), &ep) + return ep, err +} diff --git a/terratest/utils/utils.go b/terratest/utils/utils.go index 13cbbbb288..9c882dc1fc 100644 --- a/terratest/utils/utils.go +++ b/terratest/utils/utils.go @@ -39,6 +39,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const DefaultRetries = 120 + // GetIngressIPs returns slice of IP's related to ingress func GetIngressIPs(t *testing.T, options *k8s.KubectlOptions, ingressName string) []string { var ingressIPs []string @@ -139,17 +141,17 @@ func InstallPodinfo(t *testing.T, options *k8s.KubectlOptions, settings TestSett LabelSelector: "app.kubernetes.io/name=frontend-podinfo", } - k8s.WaitUntilNumPodsCreated(t, options, testAppFilter, 1, 120, 1*time.Second) + k8s.WaitUntilNumPodsCreated(t, options, testAppFilter, 1, DefaultRetries, 1*time.Second) var testAppPods []corev1.Pod testAppPods = k8s.ListPods(t, options, testAppFilter) for _, pod := range testAppPods { - k8s.WaitUntilPodAvailable(t, options, pod.Name, 120, 1*time.Second) + k8s.WaitUntilPodAvailable(t, options, pod.Name, DefaultRetries, 1*time.Second) } - k8s.WaitUntilServiceAvailable(t, options, "frontend-podinfo", 120, 1*time.Second) + k8s.WaitUntilServiceAvailable(t, options, "frontend-podinfo", DefaultRetries, 1*time.Second) } @@ -157,7 +159,7 @@ func CreateGslbWithHealthyApp(t *testing.T, options *k8s.KubectlOptions, setting CreateGslb(t, options, settings, kubeResourcePath) - k8s.WaitUntilIngressAvailable(t, options, gslbName, 120, 1*time.Second) + k8s.WaitUntilIngressAvailable(t, options, gslbName, DefaultRetries, 1*time.Second) ingress := k8s.GetIngress(t, options, gslbName) require.Equal(t, ingress.Name, gslbName) @@ -183,7 +185,7 @@ func AssertGslbStatus(t *testing.T, options *k8s.KubectlOptions, gslbName, servi _, err := DoWithRetryWaitingForValueE( t, "Wait for expected ServiceHealth status...", - 120, + DefaultRetries, 1*time.Second, actualHealthStatus, expectedHealthStatus)