From c18134dcd54e1d4b67e4d5a38cc4812aaadc11ff Mon Sep 17 00:00:00 2001 From: abaguas Date: Thu, 4 Jul 2024 11:56:14 +0200 Subject: [PATCH] istio integration Signed-off-by: Andre Baptista Aguas --- .golangci.yaml | 23 +- Makefile | 40 +++- api/v1beta1/gslb_types.go | 2 + api/v1beta1/zz_generated.deepcopy.go | 1 + chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml | 51 ++++ chart/k8gb/templates/role.yaml | 11 + chart/k8gb/values.schema.json | 90 ++++--- chart/k8gb/values.yaml | 11 +- controllers/gslb_controller_reconciliation.go | 6 +- .../gslb_controller_reconciliation_test.go | 8 +- controllers/mocks/refresolver_mock.go | 9 +- controllers/providers/dns/external_test.go | 2 +- .../refresolver/{ => ingress}/ingress.go | 49 ++-- .../refresolver/ingress/ingress_test.go | 135 +++++++++++ .../testdata/ingress_multiple_ips.yaml | 26 ++ .../testdata/ingress_multiple_servers.yaml | 1 - .../ingress/testdata/ingress_no_ips.yaml | 24 ++ controllers/refresolver/ingress_test.go | 222 ------------------ .../istiovirtualservice.go | 204 ++++++++++++++++ .../istiovirtualservice_test.go | 150 ++++++++++++ .../testdata/istio_service_multiple_ips.yaml | 21 ++ .../testdata/istio_service_no_ips.yaml | 19 ++ .../istio_virtualservice_multiple_hosts.yaml | 19 ++ .../istio_virtualservice_multiple_routes.yaml | 26 ++ controllers/refresolver/refresolver.go | 32 +-- controllers/refresolver/refresolver_test.go | 104 ++++++++ ...bedded.yaml => gslb_ingress_embedded.yaml} | 6 +- ...nced.yaml => gslb_ingress_referenced.yaml} | 4 +- ...gslb_ingress_referenced_and_embedded.yaml} | 8 +- .../refresolver/testdata/gslb_istio.yaml | 14 ++ .../testdata/ingress_embedded.yaml | 10 +- .../testdata/ingress_referenced.yaml | 9 +- .../refresolver/testdata/istio_gateway.yaml | 15 ++ .../refresolver/testdata/istio_service.yaml | 20 ++ .../testdata/istio_virtualservice.yaml | 18 ++ controllers/tracing/tracing.go | 4 +- controllers/utils/yaml.go | 121 +++++++++- ...oss_v1beta1_gslb_cr_failover_ingress.yaml} | 2 +- ...sa.oss_v1beta1_gslb_cr_failover_istio.yaml | 49 ++++ ...sa.oss_v1beta1_gslb_cr_notfound_istio.yaml | 50 ++++ ...s_v1beta1_gslb_cr_roundrobin_ingress.yaml} | 2 +- ....oss_v1beta1_gslb_cr_roundrobin_istio.yaml | 50 ++++ ...a.oss_v1beta1_gslb_cr_unhealthy_istio.yaml | 50 ++++ ...space.yaml => test-namespace-ingress.yaml} | 0 deploy/crds/test-namespace-istio.yaml | 6 + deploy/ingress/istio-ingress-values.yaml | 26 ++ deploy/test-apps/unhealthy-app-svc.yaml | 18 ++ deploy/test-apps/unhealthy-app.yaml | 23 ++ docs/local.md | 4 +- go.mod | 2 + go.sum | 4 + k3d/test-gslb1.yaml | 5 +- k3d/test-gslb2.yaml | 5 +- k3d/test-gslb3.yaml | 5 +- main.go | 3 +- .../examples/failover-istio-gateway.yaml | 15 ++ terratest/examples/failover-istio-gslb.yaml | 13 + .../failover-istio-virtualservice.yaml | 17 ++ .../failover-playground-istio-gateway.yaml | 15 ++ .../failover-playground-istio-gslb.yaml | 13 + ...lover-playground-istio-virtualservice.yaml | 17 ++ .../failover-playground-ref-gslb.yaml | 4 +- .../failover-playground-ref-ingress.yaml | 4 +- terratest/examples/failover-playground.yaml | 2 +- terratest/examples/failover.yaml | 14 +- .../roundrobin-weight1-istio-gateway.yaml | 15 ++ .../roundrobin-weight1-istio-gslb.yaml | 15 ++ ...undrobin-weight1-istio-virtualservice.yaml | 17 ++ .../examples/roundrobin2-istio-gateway.yaml | 15 ++ .../examples/roundrobin2-istio-gslb.yaml | 12 + .../roundrobin2-istio-virtualservice.yaml | 17 ++ .../k8gb_abstract_full_roundrobin_test.go | 24 +- .../test/k8gb_failover_playground_test.go | 47 +--- terratest/test/k8gb_full_failover_test.go | 47 +--- terratest/test/k8gb_full_roundrobin_test.go | 33 ++- terratest/test/k8gb_weight_test.go | 48 ++-- terratest/utils/extensions.go | 150 ++++++++++-- terratest/utils/utils.go | 4 +- 78 files changed, 1829 insertions(+), 548 deletions(-) rename controllers/refresolver/{ => ingress}/ingress.go (73%) create mode 100644 controllers/refresolver/ingress/ingress_test.go create mode 100644 controllers/refresolver/ingress/testdata/ingress_multiple_ips.yaml rename controllers/refresolver/{ => ingress}/testdata/ingress_multiple_servers.yaml (97%) create mode 100644 controllers/refresolver/ingress/testdata/ingress_no_ips.yaml delete mode 100644 controllers/refresolver/ingress_test.go create mode 100644 controllers/refresolver/istiovirtualservice/istiovirtualservice.go create mode 100644 controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go create mode 100644 controllers/refresolver/istiovirtualservice/testdata/istio_service_multiple_ips.yaml create mode 100644 controllers/refresolver/istiovirtualservice/testdata/istio_service_no_ips.yaml create mode 100644 controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_hosts.yaml create mode 100644 controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_routes.yaml create mode 100644 controllers/refresolver/refresolver_test.go rename controllers/refresolver/testdata/{gslb_embedded.yaml => gslb_ingress_embedded.yaml} (78%) rename controllers/refresolver/testdata/{gslb_referenced.yaml => gslb_ingress_referenced.yaml} (79%) rename controllers/refresolver/testdata/{gslb_referenced_and_embedded.yaml => gslb_ingress_referenced_and_embedded.yaml} (75%) create mode 100644 controllers/refresolver/testdata/gslb_istio.yaml create mode 100644 controllers/refresolver/testdata/istio_gateway.yaml create mode 100644 controllers/refresolver/testdata/istio_service.yaml create mode 100644 controllers/refresolver/testdata/istio_virtualservice.yaml rename deploy/crds/{k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml => k8gb.absa.oss_v1beta1_gslb_cr_failover_ingress.yaml} (95%) create mode 100644 deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_istio.yaml create mode 100644 deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_notfound_istio.yaml rename deploy/crds/{k8gb.absa.oss_v1beta1_gslb_cr.yaml => k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml} (98%) create mode 100644 deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_istio.yaml create mode 100644 deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_unhealthy_istio.yaml rename deploy/crds/{test-namespace.yaml => test-namespace-ingress.yaml} (100%) create mode 100644 deploy/crds/test-namespace-istio.yaml create mode 100644 deploy/ingress/istio-ingress-values.yaml create mode 100644 terratest/examples/failover-istio-gateway.yaml create mode 100644 terratest/examples/failover-istio-gslb.yaml create mode 100644 terratest/examples/failover-istio-virtualservice.yaml create mode 100644 terratest/examples/failover-playground-istio-gateway.yaml create mode 100644 terratest/examples/failover-playground-istio-gslb.yaml create mode 100644 terratest/examples/failover-playground-istio-virtualservice.yaml create mode 100644 terratest/examples/roundrobin-weight1-istio-gateway.yaml create mode 100644 terratest/examples/roundrobin-weight1-istio-gslb.yaml create mode 100644 terratest/examples/roundrobin-weight1-istio-virtualservice.yaml create mode 100644 terratest/examples/roundrobin2-istio-gateway.yaml create mode 100644 terratest/examples/roundrobin2-istio-gslb.yaml create mode 100644 terratest/examples/roundrobin2-istio-virtualservice.yaml diff --git a/.golangci.yaml b/.golangci.yaml index e9df0f870f..4a9f7ef5bf 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -30,17 +30,17 @@ linters: - dupl - gochecknoinits - # don't enable: - # - golint # deprecated - # - whitespace - # - funlen - # - exhaustive - # - gomnd - # - gofmt - # - deadcode # deprecated since v1.49.0 - # - structcheck # deprecated since v1.49.0 - # - varcheck # deprecated since v1.49.0 - # - rowserrcheck # rowserrcheck is disabled because of generics. You can track the evolution of the generics support by following the https://github.com/golangci/golangci-lint/issues/2649 + # don't enable: + # - golint # deprecated + # - whitespace + # - funlen + # - exhaustive + # - gomnd + # - gofmt + # - deadcode # deprecated since v1.49.0 + # - structcheck # deprecated since v1.49.0 + # - varcheck # deprecated since v1.49.0 + # - rowserrcheck # rowserrcheck is disabled because of generics. You can track the evolution of the generics support by following the https://github.com/golangci/golangci-lint/issues/2649 run: deadline: 3m @@ -55,6 +55,7 @@ linters-settings: main: allow: - $gostd + - istio.io/client-go - k8s.io/apimachinery - k8s.io/client-go - k8s.io/api diff --git a/Makefile b/Makefile index 227620798b..0dae1e9554 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,7 @@ STABLE_VERSION := "stable" BUNDLE_IMG ?= controller-bundle:$(VERSION) NGINX_INGRESS_VALUES_PATH ?= deploy/ingress/nginx-ingress-values.yaml +ISTIO_INGRESS_VALUES_PATH ?= deploy/ingress/istio-ingress-values.yaml # options for 'bundle-build' ifneq ($(origin CHANNELS), undefined) @@ -188,6 +189,20 @@ deploy-local-cluster: helm -n k8gb upgrade -i nginx-ingress nginx-stable/ingress-nginx \ --version 4.0.15 -f $(NGINX_INGRESS_VALUES_PATH) + @echo -e "\n$(YELLOW)Install Istio CRDs $(NC)" + kubectl create namespace istio-system + helm repo add --force-update istio https://istio-release.storage.googleapis.com/charts + helm repo update + helm upgrade -i istio-base istio/base -n istio-system + + @echo -e "\n$(YELLOW)Install Istiod $(NC)" + helm upgrade -i istiod istio/istiod -n istio-system --wait + + @echo -e "\n$(YELLOW)Install Istio Ingress Gateway $(NC)" + kubectl create namespace istio-ingress + helm upgrade -i istio-ingressgateway istio/gateway -n istio-ingress \ + -f $(ISTIO_INGRESS_VALUES_PATH) + @if [ "$(DEPLOY_APPS)" = true ]; then $(MAKE) deploy-test-apps ; fi @echo -e "\n$(YELLOW)Wait until Ingress controller is ready $(NC)" @@ -198,9 +213,15 @@ deploy-local-cluster: .PHONY: deploy-test-apps deploy-test-apps: ## Deploy Podinfo (example app) and Apply Gslb Custom Resources @echo -e "\n$(YELLOW)Deploy GSLB cr $(NC)" - kubectl apply -f deploy/crds/test-namespace.yaml - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml) - $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml) + kubectl apply -f deploy/crds/test-namespace-ingress.yaml + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml) + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_ingress.yaml) + + kubectl apply -f deploy/crds/test-namespace-istio.yaml + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_istio.yaml) + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_istio.yaml) + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_notfound_istio.yaml) + $(call apply-cr,deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_unhealthy_istio.yaml) @echo -e "\n$(YELLOW)Deploy podinfo $(NC)" kubectl apply -f deploy/test-apps @@ -210,6 +231,11 @@ deploy-test-apps: ## Deploy Podinfo (example app) and Apply Gslb Custom Resource --set image.repository="$(PODINFO_IMAGE_REPO)" \ podinfo/podinfo \ --version 5.1.1 + helm upgrade --install frontend --namespace test-gslb-istio -f deploy/test-apps/podinfo/podinfo-values.yaml \ + --set ui.message="`$(call get-cluster-geo-tag)`" \ + --set image.repository="$(PODINFO_IMAGE_REPO)" \ + podinfo/podinfo \ + --version 5.1.1 .PHONY: deploy-kuar-app deploy-kuar-app: @@ -342,11 +368,11 @@ docker-push: test .PHONY: init-failover init-failover: - $(call init-test-strategy, "deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml") + $(call init-test-strategy, "deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_ingress.yaml") .PHONY: init-round-robin init-round-robin: - $(call init-test-strategy, "deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml") + $(call init-test-strategy, "deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml") # creates infoblox secret in current cluster .PHONY: infoblox-secret @@ -554,9 +580,9 @@ endef define debug $(call manifest) - kubectl apply -f deploy/crds/test-namespace.yaml + kubectl apply -f deploy/crds/test-namespace-ingress.yaml kubectl apply -f ./chart/k8gb/templates/k8gb.absa.oss_gslbs.yaml - kubectl apply -f ./deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml + kubectl apply -f ./deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml dlv $1 endef diff --git a/api/v1beta1/gslb_types.go b/api/v1beta1/gslb_types.go index 2a913fc26b..54ae5eaee9 100644 --- a/api/v1beta1/gslb_types.go +++ b/api/v1beta1/gslb_types.go @@ -45,6 +45,8 @@ type Strategy struct { type ResourceRef struct { // Ingress selects a kubernetes.networking.k8s.io/v1.Ingress resource Ingress metav1.LabelSelector `json:"ingress,omitempty"` + // IstioVirtualService selects a networking.istio.io/v1.VirtualService resource + IstioVirtualService metav1.LabelSelector `json:"istioVirtualService,omitempty"` } // GslbSpec defines the desired state of Gslb diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 2bfaabbcee..19511be463 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -222,6 +222,7 @@ func (in *NamespacedName) DeepCopy() *NamespacedName { func (in *ResourceRef) DeepCopyInto(out *ResourceRef) { *out = *in in.Ingress.DeepCopyInto(&out.Ingress) + in.IstioVirtualService.DeepCopyInto(&out.IstioVirtualService) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceRef. diff --git a/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml b/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml index 693d89a85a..cf196c9574 100644 --- a/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml +++ b/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml @@ -346,11 +346,62 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + istioVirtualService: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic required: - key - operator type: object type: array + x-kubernetes-list-type: atomic matchLabels: additionalProperties: type: string diff --git a/chart/k8gb/templates/role.yaml b/chart/k8gb/templates/role.yaml index 15502a9013..2455c5b263 100644 --- a/chart/k8gb/templates/role.yaml +++ b/chart/k8gb/templates/role.yaml @@ -56,4 +56,15 @@ rules: verbs: - update {{- end }} +{{- if .Values.istio.enabled }} +- apiGroups: + - networking.istio.io + resources: + - virtualservices + - gateways + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} {{- end }} diff --git a/chart/k8gb/values.schema.json b/chart/k8gb/values.schema.json index 17c82f8d95..7cfcf4975f 100644 --- a/chart/k8gb/values.schema.json +++ b/chart/k8gb/values.schema.json @@ -32,7 +32,7 @@ }, "azuredns": { "$ref": "#/definitions/AzureDNS" - }, + }, "cloudflare": { "$ref": "#/definitions/Cloudflare" }, @@ -41,6 +41,9 @@ }, "tracing": { "$ref": "#/definitions/Tracing" + }, + "istio": { + "$ref": "#/definitions/Istio" } } }, @@ -628,42 +631,42 @@ }, "createAuthSecret": { "type": "object", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean" - }, - "tenantId": { - "type": "string", - "minLength": 1 - }, - "subscriptionId": { - "type": "string", - "minLength": 1 - }, - "resourceGroup": { - "type": "string", - "minLength": 1 - }, - "aadClientId": { - "type": "string" - }, - "aadClientSecret": { - "type": "string" - }, - "useManagedIdentityExtension": { - "type": "boolean", - "default": false - }, - "userAssignedIdentityID": { - "type": "string" - }, - "useWorkloadIdentityExtension": { - "type": "boolean", - "default": false - } - } - } + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + }, + "tenantId": { + "type": "string", + "minLength": 1 + }, + "subscriptionId": { + "type": "string", + "minLength": 1 + }, + "resourceGroup": { + "type": "string", + "minLength": 1 + }, + "aadClientId": { + "type": "string" + }, + "aadClientSecret": { + "type": "string" + }, + "useManagedIdentityExtension": { + "type": "boolean", + "default": false + }, + "userAssignedIdentityID": { + "type": "string" + }, + "useWorkloadIdentityExtension": { + "type": "boolean", + "default": false + } + } + } }, "required": [ "enabled" @@ -729,6 +732,19 @@ "enabled" ], "title": "Tracing" + }, + "Istio": { + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "title": "Istio" } } } diff --git a/chart/k8gb/values.yaml b/chart/k8gb/values.yaml index 25da0e501e..50a4d46146 100644 --- a/chart/k8gb/values.yaml +++ b/chart/k8gb/values.yaml @@ -21,8 +21,8 @@ k8gb: edgeDNSZone: "example.com" # main zone which would contain gslb zone to delegate # -- host/ip[:port] format is supported here where port defaults to 53 edgeDNSServers: - # -- use this DNS server as a main resolver to enable cross k8gb DNS based communication - - "1.1.1.1" + # -- use this DNS server as a main resolver to enable cross k8gb DNS based communication + - "1.1.1.1" # -- used for places where we need to distinguish between different Gslb instances clusterGeoTag: "eu" # -- comma-separated list of external gslb geo tags to pair with @@ -171,13 +171,12 @@ azuredns: # -- Azure client secret that is associated with the Service Principal. aadClientSecret: myAadClientSecret # -- Use either AKS Kubelet Identity or AAD Pod Identities - useManagedIdentityExtension : false + useManagedIdentityExtension: false # -- Client id from the Managed identitty when using the AAD Pod Identities userAssignedIdentityID: myUserAssignedIdentityID # -- Use AKS workload identity extension useWorkloadIdentityExtension: false - cloudflare: # -- Enable Cloudflare provider enabled: false @@ -224,3 +223,7 @@ tracing: repository: jaegertracing/all-in-one tag: 1.59.0 pullPolicy: Always + +istio: + # -- install istio RBAC + enabled: true diff --git a/controllers/gslb_controller_reconciliation.go b/controllers/gslb_controller_reconciliation.go index 45c94f8a68..26315c341a 100644 --- a/controllers/gslb_controller_reconciliation.go +++ b/controllers/gslb_controller_reconciliation.go @@ -95,6 +95,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } log.Debug(). Str("gslb", gslb.Name). + Str("namespace", gslb.Namespace). Interface("strategy", gslb.Spec.Strategy). Msg("Resolved strategy") // == Finalizer business == @@ -143,7 +144,8 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } // == Ingress ========== - if reflect.DeepEqual(gslb.Spec.ResourceRef.Ingress, metav1.LabelSelector{}) { + if reflect.DeepEqual(gslb.Spec.ResourceRef.Ingress, metav1.LabelSelector{}) && + reflect.DeepEqual(gslb.Spec.ResourceRef.IstioVirtualService, metav1.LabelSelector{}) { ingress, err := r.gslbIngress(gslb) if err != nil { m.IncrementError(gslb) @@ -170,7 +172,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl. } gslb.Status.Servers = servers - loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Client, r.Config.EdgeDNSServers) + loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Config.EdgeDNSServers) if err != nil { m.IncrementError(gslb) return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err)) diff --git a/controllers/gslb_controller_reconciliation_test.go b/controllers/gslb_controller_reconciliation_test.go index f1facb61bd..4dce291875 100644 --- a/controllers/gslb_controller_reconciliation_test.go +++ b/controllers/gslb_controller_reconciliation_test.go @@ -71,7 +71,7 @@ type testSettings struct { assistant assistant.Assistant } -var crSampleYaml = "../deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml" +var crSampleYaml = "../deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml" var predefinedConfig = depresolver.Config{ ReconcileRequeueSeconds: 30, @@ -1190,7 +1190,7 @@ func TestGslbRemoveDefaultFinalizer(t *testing.T) { // assert err = settings.client.Get(context.TODO(), settings.request.NamespacedName, gslb) - require.EqualError(t, err, "gslbs.k8gb.absa.oss \"test-gslb\" not found") + require.EqualError(t, err, "gslbs.k8gb.absa.oss \"roundrobin-ingress\" not found") assert.Len(t, gslb.Finalizers, 0) }) } @@ -1212,7 +1212,7 @@ func TestGslbRemoveWithFinalizer(t *testing.T) { // assert err = settings.client.Get(context.TODO(), settings.request.NamespacedName, gslb) - require.EqualError(t, err, "gslbs.k8gb.absa.oss \"test-gslb\" not found") + require.EqualError(t, err, "gslbs.k8gb.absa.oss \"roundrobin-ingress\" not found") assert.Len(t, gslb.Finalizers, 0) }) } @@ -1234,7 +1234,7 @@ func TestGslbRemoveBothFinalizers(t *testing.T) { // assert err = settings.reconciler.Get(context.TODO(), settings.request.NamespacedName, gslb) - require.EqualError(t, err, "gslbs.k8gb.absa.oss \"test-gslb\" not found") + require.EqualError(t, err, "gslbs.k8gb.absa.oss \"roundrobin-ingress\" not found") assert.Len(t, gslb.Finalizers, 0) }) } diff --git a/controllers/mocks/refresolver_mock.go b/controllers/mocks/refresolver_mock.go index 99a3db8a33..2b5ce011f8 100644 --- a/controllers/mocks/refresolver_mock.go +++ b/controllers/mocks/refresolver_mock.go @@ -33,7 +33,6 @@ import ( v1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" utils "github.com/k8gb-io/k8gb/controllers/utils" gomock "go.uber.org/mock/gomock" - client "sigs.k8s.io/controller-runtime/pkg/client" ) // MockGslbReferenceResolver is a mock of GslbReferenceResolver interface. @@ -60,18 +59,18 @@ func (m *MockGslbReferenceResolver) EXPECT() *MockGslbReferenceResolverMockRecor } // GetGslbExposedIPs mocks base method. -func (m *MockGslbReferenceResolver) GetGslbExposedIPs(arg0 client.Client, arg1 utils.DNSList) ([]string, error) { +func (m *MockGslbReferenceResolver) GetGslbExposedIPs(arg0 utils.DNSList) ([]string, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetGslbExposedIPs", arg0, arg1) + ret := m.ctrl.Call(m, "GetGslbExposedIPs", arg0) ret0, _ := ret[0].([]string) ret1, _ := ret[1].(error) return ret0, ret1 } // GetGslbExposedIPs indicates an expected call of GetGslbExposedIPs. -func (mr *MockGslbReferenceResolverMockRecorder) GetGslbExposedIPs(arg0, arg1 any) *gomock.Call { +func (mr *MockGslbReferenceResolverMockRecorder) GetGslbExposedIPs(arg0 any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGslbExposedIPs", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetGslbExposedIPs), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetGslbExposedIPs", reflect.TypeOf((*MockGslbReferenceResolver)(nil).GetGslbExposedIPs), arg0) } // GetServers mocks base method. diff --git a/controllers/providers/dns/external_test.go b/controllers/providers/dns/external_test.go index 3ca197cdd8..fa4ad3ecba 100644 --- a/controllers/providers/dns/external_test.go +++ b/controllers/providers/dns/external_test.go @@ -72,7 +72,7 @@ var a = struct { K8gbNamespace: "k8gb", }, Gslb: func() *k8gbv1beta1.Gslb { - var crSampleYaml = "../../../deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml" + var crSampleYaml = "../../../deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml" gslbYaml, _ := os.ReadFile(crSampleYaml) gslb, _ := utils2.YamlToGslb(gslbYaml) gslb.Status.LoadBalancer.ExposedIPs = targetIPs diff --git a/controllers/refresolver/ingress.go b/controllers/refresolver/ingress/ingress.go similarity index 73% rename from controllers/refresolver/ingress.go rename to controllers/refresolver/ingress/ingress.go index 1ca30e6977..3eb59119f4 100644 --- a/controllers/refresolver/ingress.go +++ b/controllers/refresolver/ingress/ingress.go @@ -1,4 +1,4 @@ -package refresolver +package ingress /* Copyright 2022 The k8gb Contributors. @@ -18,30 +18,13 @@ limitations under the License. Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ -/* -Copyright 2024 The k8gb Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic -*/ - import ( "context" "fmt" "reflect" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/logging" "github.com/k8gb-io/k8gb/controllers/utils" netv1 "k8s.io/api/networking/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -50,12 +33,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -type IngressReferenceResolver struct { +var log = logging.Logger() + +type ReferenceResolver struct { ingress *netv1.Ingress } -// NewIngressReferenceResolver creates a reference resolver capable of understanding ingresses.networking.k8s.io resources -func NewIngressReferenceResolver(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (*IngressReferenceResolver, error) { +// NewReferenceResolver creates a reference resolver capable of understanding ingresses.networking.k8s.io resources +func NewReferenceResolver(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (*ReferenceResolver, error) { ingressList, err := getGslbIngressRef(gslb, k8sClient) if err != nil { return nil, err @@ -78,7 +63,7 @@ func NewIngressReferenceResolver(gslb *k8gbv1beta1.Gslb, k8sClient client.Client return nil, fmt.Errorf("exactly one Ingress resource expected but %d were found", len(ingressList)) } - return &IngressReferenceResolver{ + return &ReferenceResolver{ ingress: &ingressList[0], }, nil } @@ -142,11 +127,11 @@ func getGslbIngressEmbedded(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (*n return ingress, nil } -// GetServers retrieves the backend servers referenced by the GSLB -func (irr *IngressReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { +// GetServers retrieves the GSLB server configuration from the gateway resource +func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { servers := []*k8gbv1beta1.Server{} - for _, rule := range irr.ingress.Spec.Rules { + for _, rule := range rr.ingress.Spec.Rules { server := &k8gbv1beta1.Server{ Host: rule.Host, Services: []*k8gbv1beta1.NamespacedName{}, @@ -154,7 +139,7 @@ func (irr *IngressReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) for _, path := range rule.HTTP.Paths { if path.Backend.Service == nil || path.Backend.Service.Name == "" { log.Warn(). - Str("ingress", irr.ingress.Name). + Str("ingress", rr.ingress.Name). Interface("service", path.Backend.Service). Msg("Malformed service definition") continue @@ -162,7 +147,7 @@ func (irr *IngressReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) server.Services = append(server.Services, &k8gbv1beta1.NamespacedName{ Name: path.Backend.Service.Name, - Namespace: irr.ingress.Namespace, + Namespace: rr.ingress.Namespace, }) } servers = append(servers, server) @@ -172,10 +157,10 @@ func (irr *IngressReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) } // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB -func (irr *IngressReferenceResolver) GetGslbExposedIPs(_ client.Client, edgeDNSServers utils.DNSList) ([]string, error) { +func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) { gslbIngressIPs := []string{} - for _, ip := range irr.ingress.Status.LoadBalancer.Ingress { + for _, ip := range rr.ingress.Status.LoadBalancer.Ingress { if len(ip.IP) > 0 { gslbIngressIPs = append(gslbIngressIPs, ip.IP) } @@ -191,7 +176,3 @@ func (irr *IngressReferenceResolver) GetGslbExposedIPs(_ client.Client, edgeDNSS return gslbIngressIPs, nil } - -func (irr *IngressReferenceResolver) getIngress() *netv1.Ingress { - return irr.ingress -} diff --git a/controllers/refresolver/ingress/ingress_test.go b/controllers/refresolver/ingress/ingress_test.go new file mode 100644 index 0000000000..e288101e81 --- /dev/null +++ b/controllers/refresolver/ingress/ingress_test.go @@ -0,0 +1,135 @@ +package ingress + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "testing" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/utils" + "github.com/stretchr/testify/assert" +) + +func TestGetServers(t *testing.T) { + var tests = []struct { + name string + ingressFile string + expectedServers []*k8gbv1beta1.Server + }{ + { + name: "single server", + ingressFile: "../testdata/ingress_referenced.yaml", + expectedServers: []*k8gbv1beta1.Server{ + { + Host: "ingress-referenced.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "ingress-referenced", + Namespace: "test-gslb", + }, + }, + }, + }, + }, + { + name: "multiple servers", + ingressFile: "./testdata/ingress_multiple_servers.yaml", + expectedServers: []*k8gbv1beta1.Server{ + { + Host: "h1.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "s1", + Namespace: "test-gslb", + }, + }, + }, + { + Host: "h2.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "ss1", + Namespace: "test-gslb", + }, + { + Name: "ss2", + Namespace: "test-gslb", + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + ingress := utils.FileToIngress(test.ingressFile) + resolver := ReferenceResolver{ + ingress: ingress, + } + + // act + servers, err := resolver.GetServers() + assert.NoError(t, err) + + // assert + assert.Equal(t, test.expectedServers, servers) + }) + } +} + +func TestGetGslbExposedIPs(t *testing.T) { + var tests = []struct { + name string + ingressYaml string + expectedIPs []string + }{ + { + name: "no exposed IPs", + ingressYaml: "./testdata/ingress_no_ips.yaml", + expectedIPs: []string{}, + }, + { + name: "single exposed IP", + ingressYaml: "../testdata/ingress_referenced.yaml", + expectedIPs: []string{"10.0.0.1"}, + }, + { + name: "multiple exposed IPs", + ingressYaml: "./testdata/ingress_multiple_ips.yaml", + expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + ingress := utils.FileToIngress(test.ingressYaml) + resolver := ReferenceResolver{ + ingress: ingress, + } + + // act + IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{}) + assert.NoError(t, err) + + // assert + assert.Equal(t, test.expectedIPs, IPs) + }) + } +} diff --git a/controllers/refresolver/ingress/testdata/ingress_multiple_ips.yaml b/controllers/refresolver/ingress/testdata/ingress_multiple_ips.yaml new file mode 100644 index 0000000000..3f3dc98d84 --- /dev/null +++ b/controllers/refresolver/ingress/testdata/ingress_multiple_ips.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + app: ingress-referenced + name: ingress-referenced + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: ingress-referenced.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: referenced + port: + name: http +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 + - ip: 10.0.0.2 diff --git a/controllers/refresolver/testdata/ingress_multiple_servers.yaml b/controllers/refresolver/ingress/testdata/ingress_multiple_servers.yaml similarity index 97% rename from controllers/refresolver/testdata/ingress_multiple_servers.yaml rename to controllers/refresolver/ingress/testdata/ingress_multiple_servers.yaml index 5066d8e995..49ca038a24 100644 --- a/controllers/refresolver/testdata/ingress_multiple_servers.yaml +++ b/controllers/refresolver/ingress/testdata/ingress_multiple_servers.yaml @@ -43,4 +43,3 @@ status: loadBalancer: ingress: - ip: 10.0.0.1 - - ip: 10.0.0.2 diff --git a/controllers/refresolver/ingress/testdata/ingress_no_ips.yaml b/controllers/refresolver/ingress/testdata/ingress_no_ips.yaml new file mode 100644 index 0000000000..7969b2d7c8 --- /dev/null +++ b/controllers/refresolver/ingress/testdata/ingress_no_ips.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + labels: + app: ingress-referenced + name: ingress-referenced + namespace: test-gslb + resourceVersion: "999" +spec: + ingressClassName: nginx + rules: + - host: ingress-referenced.cloud.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: referenced + port: + name: http +status: + loadBalancer: + ingress: diff --git a/controllers/refresolver/ingress_test.go b/controllers/refresolver/ingress_test.go deleted file mode 100644 index 62a7e94c30..0000000000 --- a/controllers/refresolver/ingress_test.go +++ /dev/null @@ -1,222 +0,0 @@ -package refresolver - -/* -Copyright 2022 The k8gb Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic -*/ - -/* -Copyright 2024 The k8gb Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic -*/ - -import ( - "fmt" - "os" - "testing" - - k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" - "github.com/k8gb-io/k8gb/controllers/utils" - "github.com/stretchr/testify/assert" - netv1 "k8s.io/api/networking/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/kubernetes/scheme" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -func TestGetIngress(t *testing.T) { - var tests = []struct { - name string - gslbYaml string - expectedIngressYaml string - expectedError error - }{ - { - name: "embedded ingress", - gslbYaml: "./testdata/gslb_embedded.yaml", - expectedIngressYaml: "./testdata/ingress_embedded.yaml", - expectedError: nil, - }, - { - name: "referenced ingress", - gslbYaml: "./testdata/gslb_referenced.yaml", - expectedIngressYaml: "./testdata/ingress_referenced.yaml", - expectedError: nil, - }, - { - name: "referenced and embedded ingress", - gslbYaml: "./testdata/gslb_referenced_and_embedded.yaml", - expectedIngressYaml: "", - expectedError: fmt.Errorf("exactly one Ingress resource expected but 2 were found"), - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // arrange - cl, gslb := getTestContext(test.gslbYaml) - - // act - resolver, err := New(gslb, cl) - assert.Equal(t, test.expectedError, err) - - ingressResolver, ok := resolver.(*IngressReferenceResolver) - assert.True(t, ok, "referenced resolver is of type Ingress") - - // assert - if test.expectedIngressYaml != "" { - expectedIngress := fileToIngress(test.expectedIngressYaml) - assert.Equal(t, expectedIngress, ingressResolver.getIngress()) - } - }) - } -} - -func TestGetServers(t *testing.T) { - var tests = []struct { - name string - ingressYaml string - expectedServers []*k8gbv1beta1.Server - }{ - { - name: "multiple servers", - ingressYaml: "./testdata/ingress_multiple_servers.yaml", - expectedServers: []*k8gbv1beta1.Server{ - { - Host: "h1.cloud.example.com", - Services: []*k8gbv1beta1.NamespacedName{ - { - Name: "s1", - Namespace: "test-gslb", - }, - }, - }, - { - Host: "h2.cloud.example.com", - Services: []*k8gbv1beta1.NamespacedName{ - { - Name: "ss1", - Namespace: "test-gslb", - }, - { - Name: "ss2", - Namespace: "test-gslb", - }, - }, - }, - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // arrange - ingress := fileToIngress(test.ingressYaml) - resolver := IngressReferenceResolver{ - ingress: ingress, - } - - // act - servers, err := resolver.GetServers() - assert.NoError(t, err) - - // assert - assert.Equal(t, test.expectedServers, servers) - }) - } -} - -func TestGetGslbExposedIPs(t *testing.T) { - var tests = []struct { - name string - ingressYaml string - expectedIPs []string - }{ - { - name: "multiple exposed IPs", - ingressYaml: "./testdata/ingress_multiple_servers.yaml", - expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, - }, - { - name: "no exposed IPs", - ingressYaml: "./testdata/ingress_referenced.yaml", - expectedIPs: []string{}, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - // arrange - ingress := fileToIngress(test.ingressYaml) - resolver := IngressReferenceResolver{ - ingress: ingress, - } - - // act - IPs, err := resolver.GetGslbExposedIPs(nil, []utils.DNSServer{}) - assert.NoError(t, err) - - // assert - assert.Equal(t, test.expectedIPs, IPs) - }) - } -} - -func getTestContext(gslbData string) (client.Client, *k8gbv1beta1.Gslb) { - gslbYaml, err := os.ReadFile(gslbData) - if err != nil { - panic(fmt.Errorf("can't open example CR file: %s", gslbData)) - } - gslb, err := utils.YamlToGslb(gslbYaml) - if err != nil { - panic(err) - } - - objs := []runtime.Object{ - gslb, - fileToIngress("./testdata/ingress_embedded.yaml"), - fileToIngress("./testdata/ingress_referenced.yaml"), - } - // Register operator types with the runtime scheme. - s := scheme.Scheme - s.AddKnownTypes(k8gbv1beta1.GroupVersion, gslb) - cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build() - - return cl, gslb -} - -func fileToIngress(ingressData string) *netv1.Ingress { - ingressYaml, err := os.ReadFile(ingressData) - if err != nil { - panic(fmt.Errorf("can't open example CR file: %s", ingressData)) - } - ingress, err := utils.YamlToIngress(ingressYaml) - if err != nil { - panic(err) - } - return ingress -} diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go new file mode 100644 index 0000000000..66f841c556 --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go @@ -0,0 +1,204 @@ +package istiovirtualservice + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "context" + "fmt" + "reflect" + "strings" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/logging" + "github.com/k8gb-io/k8gb/controllers/utils" + istio "istio.io/client-go/pkg/apis/networking/v1beta1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var log = logging.Logger() + +type ReferenceResolver struct { + virtualService *istio.VirtualService + lbService *corev1.Service +} + +// NewReferenceResolver creates a new reference resolver capable of understanding `networking.istio.io/v1` resources +func NewReferenceResolver(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (*ReferenceResolver, error) { + virtualServiceList, err := getGslbVirtualServiceRef(gslb, k8sClient) + if err != nil { + return nil, err + } + + if len(virtualServiceList) != 1 { + return nil, fmt.Errorf("exactly one VirtualService resource expected but %d were found", len(virtualServiceList)) + } + virtualService := virtualServiceList[0] + + gateway, err := getGateway(virtualService, k8sClient) + if err != nil { + return nil, err + } + + lbService, err := getLbService(gateway, k8sClient) + if err != nil { + return nil, err + } + + return &ReferenceResolver{ + virtualService: virtualService, + lbService: lbService, + }, nil +} + +// getGslbVirtualServiceRef resolves an istio virtual service resource referenced by the Gslb spec +func getGslbVirtualServiceRef(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) ([]*istio.VirtualService, error) { + virtualServiceList := &istio.VirtualServiceList{} + if reflect.DeepEqual(gslb.Spec.ResourceRef.IstioVirtualService, metav1.LabelSelector{}) { + log.Warn(). + Str("gslb", gslb.Name). + Msg("No configuration for referenced VirtualService resource") + return virtualServiceList.Items, nil + } + + selector, err := metav1.LabelSelectorAsSelector(&gslb.Spec.ResourceRef.IstioVirtualService) + if err != nil { + return nil, err + } + opts := &client.ListOptions{ + LabelSelector: selector, + } + + err = k8sClient.List(context.TODO(), virtualServiceList, opts) + if err != nil { + if errors.IsNotFound(err) { + log.Info(). + Str("gslb", gslb.Name). + Msg("Can't find referenced VirtualService resource") + } + return nil, err + } + + return virtualServiceList.Items, err +} + +// getGateway retrieves the istio gateway referenced by the istio virtual service +func getGateway(virtualService *istio.VirtualService, k8sClient client.Client) (*istio.Gateway, error) { + if len(virtualService.Spec.Gateways) != 1 { + return nil, fmt.Errorf("expected exactly one Gateway to be referenced by the VirtualService but %d were found", len(virtualService.Spec.Gateways)) + } + gatewayRef := strings.Split(virtualService.Spec.Gateways[0], "/") + gatewayNamespace := gatewayRef[0] + gatewayName := gatewayRef[1] + + gateway := &istio.Gateway{} + err := k8sClient.Get(context.TODO(), types.NamespacedName{ + Namespace: gatewayNamespace, + Name: gatewayName, + }, gateway) + if err != nil { + if errors.IsNotFound(err) { + log.Info(). + Str("gatewayNamespace", gatewayNamespace). + Str("gatewayName", gatewayName). + Msg("Can't find Gateway resource referenced by VirtualService") + } + return nil, err + } + + return gateway, nil +} + +// getLbService retrieves the kubernetes service referenced by an istio gateway +func getLbService(gateway *istio.Gateway, k8sClient client.Client) (*corev1.Service, error) { + gatewayServiceList := &corev1.ServiceList{} + opts := &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(gateway.Spec.Selector), + } + + err := k8sClient.List(context.TODO(), gatewayServiceList, opts) + if err != nil { + if errors.IsNotFound(err) { + log.Info(). + Str("gateway", gateway.Name). + Msg("Can't find any service with the gateway's selector") + } + return nil, err + } + + if len(gatewayServiceList.Items) != 1 { + return nil, fmt.Errorf("exactly one Service resource expected but %d were found", len(gatewayServiceList.Items)) + } + + gatewayService := &gatewayServiceList.Items[0] + return gatewayService, nil +} + +// GetServers retrieves the GSLB server configuration from the istio virtual service resource +func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) { + hosts := rr.virtualService.Spec.Hosts + if len(hosts) < 1 { + return nil, fmt.Errorf("can't find hosts in VirtualService %s", rr.virtualService.Name) + } + + servers := []*k8gbv1beta1.Server{} + for _, host := range hosts { + server := &k8gbv1beta1.Server{ + Host: host, + Services: []*k8gbv1beta1.NamespacedName{}, + } + for _, http := range rr.virtualService.Spec.Http { + for _, route := range http.Route { + server.Services = append(server.Services, &k8gbv1beta1.NamespacedName{ + Name: route.Destination.Host, + Namespace: rr.virtualService.Namespace, + }) + } + } + servers = append(servers, server) + + } + + return servers, nil +} + +// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB +func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) { + gslbIngressIPs := []string{} + + for _, ip := range rr.lbService.Status.LoadBalancer.Ingress { + if len(ip.IP) > 0 { + gslbIngressIPs = append(gslbIngressIPs, ip.IP) + } + if len(ip.Hostname) > 0 { + IPs, err := utils.Dig(ip.Hostname, edgeDNSServers...) + if err != nil { + log.Warn().Err(err).Msg("Dig error") + return nil, err + } + gslbIngressIPs = append(gslbIngressIPs, IPs...) + } + } + + return gslbIngressIPs, nil +} diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go new file mode 100644 index 0000000000..8938e6dd10 --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go @@ -0,0 +1,150 @@ +package istiovirtualservice + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "testing" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + "github.com/k8gb-io/k8gb/controllers/utils" + "github.com/stretchr/testify/assert" +) + +func TestGetServers(t *testing.T) { + var tests = []struct { + name string + virtualServiceFile string + expectedServers []*k8gbv1beta1.Server + }{ + { + name: "single host and route", + virtualServiceFile: "../testdata/istio_virtualservice.yaml", + expectedServers: []*k8gbv1beta1.Server{ + { + Host: "istio.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "istio", + Namespace: "test-gslb", + }, + }, + }, + }, + }, + { + name: "multiple hosts", + virtualServiceFile: "./testdata/istio_virtualservice_multiple_hosts.yaml", + expectedServers: []*k8gbv1beta1.Server{ + { + Host: "istio1.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "istio", + Namespace: "test-gslb", + }, + }, + }, + { + Host: "istio2.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "istio", + Namespace: "test-gslb", + }, + }, + }, + }, + }, + { + name: "multiple routes", + virtualServiceFile: "./testdata/istio_virtualservice_multiple_routes.yaml", + expectedServers: []*k8gbv1beta1.Server{ + { + Host: "istio.cloud.example.com", + Services: []*k8gbv1beta1.NamespacedName{ + { + Name: "istio1", + Namespace: "test-gslb", + }, + { + Name: "istio2", + Namespace: "test-gslb", + }, + }, + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + vs := utils.FileToIstioVirtualService(test.virtualServiceFile) + resolver := ReferenceResolver{ + virtualService: vs, + } + + // act + servers, err := resolver.GetServers() + assert.NoError(t, err) + + // assert + assert.Equal(t, test.expectedServers, servers) + }) + } +} + +func TestGetGslbExposedIPs(t *testing.T) { + var tests = []struct { + name string + serviceYaml string + expectedIPs []string + }{ + { + name: "no exposed IPs", + serviceYaml: "./testdata/istio_service_no_ips.yaml", + expectedIPs: []string{}, + }, + { + name: "single exposed IP", + serviceYaml: "../testdata/istio_service.yaml", + expectedIPs: []string{"10.0.0.1"}, + }, + { + name: "multiple exposed IPs", + serviceYaml: "./testdata/istio_service_multiple_ips.yaml", + expectedIPs: []string{"10.0.0.1", "10.0.0.2"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + svc := utils.FileToService(test.serviceYaml) + resolver := ReferenceResolver{ + lbService: svc, + } + + // act + IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{}) + assert.NoError(t, err) + + // assert + assert.Equal(t, test.expectedIPs, IPs) + }) + } +} diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_service_multiple_ips.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_service_multiple_ips.yaml new file mode 100644 index 0000000000..4cd55a08f1 --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/testdata/istio_service_multiple_ips.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: istio-ingress + labels: + app: istio-ingressgateway +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: istio-ingressgateway + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 + - ip: 10.0.0.2 diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_service_no_ips.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_service_no_ips.yaml new file mode 100644 index 0000000000..e7e0782b72 --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/testdata/istio_service_no_ips.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: istio-ingress + labels: + app: istio-ingressgateway +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: istio-ingressgateway + type: LoadBalancer +status: + loadBalancer: + ingress: diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_hosts.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_hosts.yaml new file mode 100644 index 0000000000..53d0e40c53 --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_hosts.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: istio + namespace: test-gslb + labels: + app: istio +spec: + gateways: + - istio-ingress/istio + hosts: + - istio1.cloud.example.com + - istio2.cloud.example.com + http: + - route: + - destination: + host: istio + port: + number: 80 diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_routes.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_routes.yaml new file mode 100644 index 0000000000..a0386959ae --- /dev/null +++ b/controllers/refresolver/istiovirtualservice/testdata/istio_virtualservice_multiple_routes.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: istio + namespace: test-gslb + labels: + app: istio +spec: + gateways: + - istio-ingress/istio + hosts: + - istio.cloud.example.com + http: + - match: + - uri: + prefix: /debug + route: + - destination: + host: istio1 + port: + number: 80 + - route: + - destination: + host: istio2 + port: + number: 80 diff --git a/controllers/refresolver/refresolver.go b/controllers/refresolver/refresolver.go index b63c19c453..95f5420c67 100644 --- a/controllers/refresolver/refresolver.go +++ b/controllers/refresolver/refresolver.go @@ -18,49 +18,33 @@ limitations under the License. Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ -/* -Copyright 2024 The k8gb Contributors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic -*/ - import ( "fmt" "reflect" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" - "github.com/k8gb-io/k8gb/controllers/logging" + "github.com/k8gb-io/k8gb/controllers/refresolver/ingress" + "github.com/k8gb-io/k8gb/controllers/refresolver/istiovirtualservice" "github.com/k8gb-io/k8gb/controllers/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) -var log = logging.Logger() - // GslbReferenceResolver resolves references to other kubernetes resources concerning ingress configuration type GslbReferenceResolver interface { - // GetServers retrieves the server configuration of the GSLB object + // GetServers retrieves GSLB the server configuration GetServers() ([]*k8gbv1beta1.Server, error) // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB - GetGslbExposedIPs(client.Client, utils.DNSList) ([]string, error) + GetGslbExposedIPs(utils.DNSList) ([]string, error) } // New creates a new GSLBReferenceResolver func New(gslb *k8gbv1beta1.Gslb, k8sClient client.Client) (GslbReferenceResolver, error) { if !reflect.DeepEqual(gslb.Spec.Ingress, k8gbv1beta1.IngressSpec{}) || !reflect.DeepEqual(gslb.Spec.ResourceRef.Ingress, metav1.LabelSelector{}) { - return NewIngressReferenceResolver(gslb, k8sClient) + return ingress.NewReferenceResolver(gslb, k8sClient) + } + if !reflect.DeepEqual(gslb.Spec.ResourceRef.IstioVirtualService, metav1.LabelSelector{}) { + return istiovirtualservice.NewReferenceResolver(gslb, k8sClient) } return nil, fmt.Errorf("incomplete gslb configuration, no ingress found") } diff --git a/controllers/refresolver/refresolver_test.go b/controllers/refresolver/refresolver_test.go new file mode 100644 index 0000000000..fbb91ff5b8 --- /dev/null +++ b/controllers/refresolver/refresolver_test.go @@ -0,0 +1,104 @@ +package refresolver + +/* +Copyright 2022 The k8gb Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic +*/ + +import ( + "fmt" + "testing" + + k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + utils "github.com/k8gb-io/k8gb/controllers/utils" + "github.com/stretchr/testify/assert" + istio "istio.io/client-go/pkg/apis/networking/v1beta1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +// TestNew tests if the RefResolver is instantiated with the correct type +func TestNew(t *testing.T) { + var tests = []struct { + name string + gslbYaml string + expectedReferenceResolverType string + expectedError error + }{ + { + name: "embedded ingress", + gslbYaml: "./testdata/gslb_ingress_embedded.yaml", + expectedReferenceResolverType: "*ingress.ReferenceResolver", + expectedError: nil, + }, + { + name: "referenced ingress", + gslbYaml: "./testdata/gslb_ingress_referenced.yaml", + expectedReferenceResolverType: "*ingress.ReferenceResolver", + expectedError: nil, + }, + { + name: "referenced istio", + gslbYaml: "./testdata/gslb_istio.yaml", + expectedReferenceResolverType: "*istiovirtualservice.ReferenceResolver", + expectedError: nil, + }, + { + name: "referenced and embedded ingress", + gslbYaml: "./testdata/gslb_ingress_referenced_and_embedded.yaml", + expectedReferenceResolverType: "", + expectedError: fmt.Errorf("exactly one Ingress resource expected but 2 were found"), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + // arrange + cl, gslb := getTestContext(test.gslbYaml) + + // act + resolver, err := New(gslb, cl) + + // assert + assert.Equal(t, test.expectedError, err) + if err == nil { + assert.Equal(t, test.expectedReferenceResolverType, fmt.Sprintf("%T", resolver)) + } + }) + } +} + +// getTextContext creates a kubernetes client with access to all objects that need to be looked up +func getTestContext(gslbFile string) (client.Client, *k8gbv1beta1.Gslb) { + gslb := utils.FileToGSLB(gslbFile) + objs := []runtime.Object{ + gslb, + utils.FileToIngress("./testdata/ingress_referenced.yaml"), + utils.FileToIngress("./testdata/ingress_embedded.yaml"), + utils.FileToIstioVirtualService("./testdata/istio_virtualservice.yaml"), + utils.FileToIstioGateway("./testdata/istio_gateway.yaml"), + utils.FileToService("./testdata/istio_service.yaml"), + } + // Register operator types with the runtime scheme. + s := scheme.Scheme + s.AddKnownTypes(k8gbv1beta1.GroupVersion, &k8gbv1beta1.Gslb{}, &k8gbv1beta1.GslbList{}) + s.AddKnownTypes(istio.SchemeGroupVersion, &istio.VirtualService{}, &istio.VirtualServiceList{}) + s.AddKnownTypes(istio.SchemeGroupVersion, &istio.Gateway{}, &istio.GatewayList{}) + cl := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build() + + return cl, gslb +} diff --git a/controllers/refresolver/testdata/gslb_embedded.yaml b/controllers/refresolver/testdata/gslb_ingress_embedded.yaml similarity index 78% rename from controllers/refresolver/testdata/gslb_embedded.yaml rename to controllers/refresolver/testdata/gslb_ingress_embedded.yaml index b199886d6c..5099a7275f 100644 --- a/controllers/refresolver/testdata/gslb_embedded.yaml +++ b/controllers/refresolver/testdata/gslb_ingress_embedded.yaml @@ -1,20 +1,20 @@ apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: - name: embedded + name: ingress-embedded namespace: test-gslb spec: ingress: ingressClassName: nginx rules: - - host: embedded.cloud.example.com + - host: ingress-embedded.cloud.example.com http: paths: - path: / pathType: Prefix backend: service: - name: embedded + name: ingress-embedded port: name: http strategy: diff --git a/controllers/refresolver/testdata/gslb_referenced.yaml b/controllers/refresolver/testdata/gslb_ingress_referenced.yaml similarity index 79% rename from controllers/refresolver/testdata/gslb_referenced.yaml rename to controllers/refresolver/testdata/gslb_ingress_referenced.yaml index 271a00ebee..51aa7ef4f9 100644 --- a/controllers/refresolver/testdata/gslb_referenced.yaml +++ b/controllers/refresolver/testdata/gslb_ingress_referenced.yaml @@ -1,13 +1,13 @@ apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: - name: referenced + name: ingress-referenced namespace: test-gslb spec: resourceRef: ingress: matchLabels: - app: referenced + app: ingress-referenced strategy: type: roundRobin splitBrainThresholdSeconds: 300 diff --git a/controllers/refresolver/testdata/gslb_referenced_and_embedded.yaml b/controllers/refresolver/testdata/gslb_ingress_referenced_and_embedded.yaml similarity index 75% rename from controllers/refresolver/testdata/gslb_referenced_and_embedded.yaml rename to controllers/refresolver/testdata/gslb_ingress_referenced_and_embedded.yaml index 1073d464dd..619d453aa6 100644 --- a/controllers/refresolver/testdata/gslb_referenced_and_embedded.yaml +++ b/controllers/refresolver/testdata/gslb_ingress_referenced_and_embedded.yaml @@ -1,24 +1,24 @@ apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: - name: embedded + name: ingress-embedded namespace: test-gslb spec: resourceRef: ingress: matchLabels: - app: referenced + app: ingress-referenced ingress: ingressClassName: nginx rules: - - host: embedded.cloud.example.com + - host: ingress-embedded.cloud.example.com http: paths: - path: / pathType: Prefix backend: service: - name: embedded + name: ingress-embedded port: name: http strategy: diff --git a/controllers/refresolver/testdata/gslb_istio.yaml b/controllers/refresolver/testdata/gslb_istio.yaml new file mode 100644 index 0000000000..7664aa4da8 --- /dev/null +++ b/controllers/refresolver/testdata/gslb_istio.yaml @@ -0,0 +1,14 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: istio + namespace: test-gslb +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: istio + strategy: + type: roundRobin + splitBrainThresholdSeconds: 300 + dnsTtlSeconds: 30 diff --git a/controllers/refresolver/testdata/ingress_embedded.yaml b/controllers/refresolver/testdata/ingress_embedded.yaml index 7b939fe47d..7a495ac30e 100644 --- a/controllers/refresolver/testdata/ingress_embedded.yaml +++ b/controllers/refresolver/testdata/ingress_embedded.yaml @@ -1,19 +1,23 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: embedded + name: ingress-embedded namespace: test-gslb resourceVersion: "999" spec: ingressClassName: nginx rules: - - host: embedded.cloud.example.com + - host: ingress-embedded.cloud.example.com http: paths: - path: / pathType: Prefix backend: service: - name: embedded + name: ingress-embedded port: name: http +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/testdata/ingress_referenced.yaml b/controllers/refresolver/testdata/ingress_referenced.yaml index e4b4e8a61b..fab04fb9f0 100644 --- a/controllers/refresolver/testdata/ingress_referenced.yaml +++ b/controllers/refresolver/testdata/ingress_referenced.yaml @@ -2,23 +2,24 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: labels: - app: referenced - name: referenced + app: ingress-referenced + name: ingress-referenced namespace: test-gslb resourceVersion: "999" spec: ingressClassName: nginx rules: - - host: referenced.cloud.example.com + - host: ingress-referenced.cloud.example.com http: paths: - path: / pathType: Prefix backend: service: - name: referenced + name: ingress-referenced port: name: http status: loadBalancer: ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/testdata/istio_gateway.yaml b/controllers/refresolver/testdata/istio_gateway.yaml new file mode 100644 index 0000000000..53aaeb2b05 --- /dev/null +++ b/controllers/refresolver/testdata/istio_gateway.yaml @@ -0,0 +1,15 @@ +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - istio.cloud.example.com + port: + name: http + number: 8080 + protocol: http diff --git a/controllers/refresolver/testdata/istio_service.yaml b/controllers/refresolver/testdata/istio_service.yaml new file mode 100644 index 0000000000..b87da87c61 --- /dev/null +++ b/controllers/refresolver/testdata/istio_service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: istio-ingressgateway + namespace: istio-ingress + labels: + app: istio-ingressgateway +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 8080 + selector: + app: istio-ingressgateway + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 10.0.0.1 diff --git a/controllers/refresolver/testdata/istio_virtualservice.yaml b/controllers/refresolver/testdata/istio_virtualservice.yaml new file mode 100644 index 0000000000..7d4a96c4d0 --- /dev/null +++ b/controllers/refresolver/testdata/istio_virtualservice.yaml @@ -0,0 +1,18 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + labels: + app: istio + name: istio + namespace: test-gslb +spec: + gateways: + - istio-ingress/istio + hosts: + - istio.cloud.example.com + http: + - route: + - destination: + host: istio + port: + number: 80 diff --git a/controllers/tracing/tracing.go b/controllers/tracing/tracing.go index b3121e4679..f50c708eee 100644 --- a/controllers/tracing/tracing.go +++ b/controllers/tracing/tracing.go @@ -31,6 +31,7 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.6.1" "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" ) const ( @@ -48,8 +49,7 @@ type Settings struct { func SetupTracing(ctx context.Context, cfg Settings, log *zerolog.Logger) (func(), trace.Tracer) { if !cfg.Enabled { log.Info().Msg("OTLP tracing is disabled") - //nolint:staticcheck - return func() {}, trace.NewNoopTracerProvider().Tracer(instrumentationName) + return func() {}, noop.NewTracerProvider().Tracer(instrumentationName) } log.Info().Msg("OTLP tracing is ON") client := otlptracehttp.NewClient( diff --git a/controllers/utils/yaml.go b/controllers/utils/yaml.go index 1d91d79096..2761488677 100644 --- a/controllers/utils/yaml.go +++ b/controllers/utils/yaml.go @@ -21,13 +21,83 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic import ( "encoding/json" + "fmt" + "os" yamlConv "github.com/ghodss/yaml" k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1" + istio "istio.io/client-go/pkg/apis/networking/v1beta1" + corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" ) -// YamlToGslb takes yaml and returns Gslb object +// FileToGSLB takes a file and returns a GSLB object +func FileToGSLB(file string) *k8gbv1beta1.Gslb { + yaml, err := os.ReadFile(file) + if err != nil { + panic(fmt.Errorf("can't open example CR file: %s", file)) + } + gslb, err := YamlToGslb(yaml) + if err != nil { + panic(err) + } + return gslb + +} + +// FileToIngress takes a file and returns an Ingress object +func FileToIngress(file string) *netv1.Ingress { + ingressYaml, err := os.ReadFile(file) + if err != nil { + panic(fmt.Errorf("can't open example CR file: %s", file)) + } + ingress, err := YamlToIngress(ingressYaml) + if err != nil { + panic(err) + } + return ingress +} + +// FileToIstioVirtualService takes a file and returns a VirtualService object +func FileToIstioVirtualService(file string) *istio.VirtualService { + yaml, err := os.ReadFile(file) + if err != nil { + panic(fmt.Errorf("can't open example CR file: %s", file)) + } + vs, err := YamlToIstioVirtualService(yaml) + if err != nil { + panic(err) + } + return vs +} + +// FileToIstioGateway takes a file and returns a Gateway object +func FileToIstioGateway(file string) *istio.Gateway { + yaml, err := os.ReadFile(file) + if err != nil { + panic(fmt.Errorf("can't open example CR file: %s", file)) + } + gw, err := YamlToIstioGateway(yaml) + if err != nil { + panic(err) + } + return gw +} + +// FileToService takes a file and returns a Service object +func FileToService(file string) *corev1.Service { + yaml, err := os.ReadFile(file) + if err != nil { + panic(fmt.Errorf("can't open example CR file: %s", file)) + } + svc, err := YamlToService(yaml) + if err != nil { + panic(err) + } + return svc +} + +// YamlToGslb takes yaml and returns a Gslb object func YamlToGslb(yaml []byte) (*k8gbv1beta1.Gslb, error) { // yamlBytes contains a []byte of my yaml job spec // convert the yaml to json @@ -60,3 +130,52 @@ func YamlToIngress(yaml []byte) (*netv1.Ingress, error) { } return ingress, nil } + +// YamlToService takes yaml and returns a Kubernetes Service object +func YamlToService(yaml []byte) (*corev1.Service, error) { + // convert the yaml to json + jsonBytes, err := yamlConv.YAMLToJSON(yaml) + if err != nil { + return &corev1.Service{}, err + } + // unmarshal the json into the kube struct + svc := &corev1.Service{} + err = json.Unmarshal(jsonBytes, &svc) + if err != nil { + return &corev1.Service{}, err + } + return svc, nil +} + +// YamlToIstioVirtualService takes yaml and returns an Istio Virtual Service object +func YamlToIstioVirtualService(yaml []byte) (*istio.VirtualService, error) { + // convert the yaml to json + jsonBytes, err := yamlConv.YAMLToJSON(yaml) + if err != nil { + return &istio.VirtualService{}, err + } + // unmarshal the json into the kube struct + vs := &istio.VirtualService{} + err = json.Unmarshal(jsonBytes, &vs) + if err != nil { + return &istio.VirtualService{}, err + } + return vs, nil +} + +// YamlToIstioGateway takes yaml and returns an Istio Gateway object +func YamlToIstioGateway(yaml []byte) (*istio.Gateway, error) { + // yamlBytes contains a []byte of my yaml job spec + // convert the yaml to json + jsonBytes, err := yamlConv.YAMLToJSON(yaml) + if err != nil { + return &istio.Gateway{}, err + } + // unmarshal the json into the kube struct + gw := &istio.Gateway{} + err = json.Unmarshal(jsonBytes, &gw) + if err != nil { + return &istio.Gateway{}, err + } + return gw, nil +} diff --git a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_ingress.yaml similarity index 95% rename from deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml rename to deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_ingress.yaml index 43fa6b458a..93309d257c 100644 --- a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml +++ b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_ingress.yaml @@ -1,7 +1,7 @@ apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: - name: test-gslb-failover + name: failover-ingress namespace: test-gslb spec: ingress: diff --git a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_istio.yaml b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_istio.yaml new file mode 100644 index 0000000000..25787e04ec --- /dev/null +++ b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_istio.yaml @@ -0,0 +1,49 @@ +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: failover-istio + namespace: test-gslb-istio + labels: + app: failover-istio +spec: + gateways: + - istio-ingress/failover-istio + hosts: + - failover-istio.cloud.example.com + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 +--- +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: failover-istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - failover-istio.cloud.example.com + port: + number: 8080 + name: http + protocol: http +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: failover-istio + namespace: test-gslb-istio +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: failover-istio + strategy: + type: failover + primaryGeoTag: eu diff --git a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_notfound_istio.yaml b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_notfound_istio.yaml new file mode 100644 index 0000000000..bc71288caa --- /dev/null +++ b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_notfound_istio.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: notfound-istio + namespace: test-gslb-istio + labels: + app: notfound-istio +spec: + gateways: + - istio-ingress/notfound-istio + hosts: + - notfound-istio.cloud.example.com + http: + - route: + - destination: + host: non-existing-app + port: + number: 9898 +--- +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: notfound-istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - notfound-istio.cloud.example.com + port: + number: 8080 + name: http + protocol: http +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: notfound-istio + namespace: test-gslb-istio +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: notfound-istio + strategy: + type: roundRobin + splitBrainThresholdSeconds: 300 + dnsTtlSeconds: 30 diff --git a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml similarity index 98% rename from deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml rename to deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml index 4731328858..e1f213ffb5 100644 --- a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml +++ b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml @@ -1,7 +1,7 @@ apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: - name: test-gslb + name: roundrobin-ingress namespace: test-gslb spec: ingress: diff --git a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_istio.yaml b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_istio.yaml new file mode 100644 index 0000000000..cfd2d856a7 --- /dev/null +++ b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_istio.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: roundrobin-istio + namespace: test-gslb-istio + labels: + app: roundrobin-istio +spec: + gateways: + - istio-ingress/roundrobin-istio + hosts: + - roundrobin-istio.cloud.example.com + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 +--- +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: roundrobin-istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - roundrobin-istio.cloud.example.com + port: + number: 8080 + name: http + protocol: http +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: roundrobin-istio + namespace: test-gslb-istio +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: roundrobin-istio + strategy: + type: roundRobin + splitBrainThresholdSeconds: 300 + dnsTtlSeconds: 30 diff --git a/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_unhealthy_istio.yaml b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_unhealthy_istio.yaml new file mode 100644 index 0000000000..ec75e4a0d2 --- /dev/null +++ b/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_unhealthy_istio.yaml @@ -0,0 +1,50 @@ +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: unhealthy-istio + namespace: test-gslb-istio + labels: + app: unhealthy-istio +spec: + gateways: + - istio-ingress/unhealthy-istio + hosts: + - unhealthy-istio.cloud.example.com + http: + - route: + - destination: + host: unhealthy-istio-app + port: + number: 80 +--- +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: unhealthy-istio + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - unhealthy-istio.cloud.example.com + port: + number: 8080 + name: http + protocol: http +--- +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: unhealthy-istio + namespace: test-gslb-istio +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: unhealthy-istio + strategy: + type: roundRobin + splitBrainThresholdSeconds: 300 + dnsTtlSeconds: 30 diff --git a/deploy/crds/test-namespace.yaml b/deploy/crds/test-namespace-ingress.yaml similarity index 100% rename from deploy/crds/test-namespace.yaml rename to deploy/crds/test-namespace-ingress.yaml diff --git a/deploy/crds/test-namespace-istio.yaml b/deploy/crds/test-namespace-istio.yaml new file mode 100644 index 0000000000..3d2388736a --- /dev/null +++ b/deploy/crds/test-namespace-istio.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: test-gslb-istio + labels: + istio-injection: enabled diff --git a/deploy/ingress/istio-ingress-values.yaml b/deploy/ingress/istio-ingress-values.yaml new file mode 100644 index 0000000000..15f624fc7c --- /dev/null +++ b/deploy/ingress/istio-ingress-values.yaml @@ -0,0 +1,26 @@ +# the ingress port on k3d is open only on the agent-0 node +# so we need to make sure the ingress controller runs on that node +affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - k3d-test-gslb1-agent-0 + - k3d-test-gslb2-agent-0 + - k3d-test-gslb3-agent-0 + - k3d-test-gslb4-agent-0 + - k3d-test-gslb5-agent-0 + - k3d-test-gslb6-agent-0 + - k3d-test-gslb7-agent-0 + - k3d-test-gslb8-agent-0 +autoscaling: + enabled: false +service: + ports: + - name: http + port: 8080 + protocol: TCP + targetPort: 8080 diff --git a/deploy/test-apps/unhealthy-app-svc.yaml b/deploy/test-apps/unhealthy-app-svc.yaml index 1994ce9780..1fbc1e471c 100644 --- a/deploy/test-apps/unhealthy-app-svc.yaml +++ b/deploy/test-apps/unhealthy-app-svc.yaml @@ -1,3 +1,4 @@ +--- apiVersion: v1 kind: Service metadata: @@ -14,3 +15,20 @@ spec: run: unhealthy-app sessionAffinity: None type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: unhealthy-app + name: unhealthy-app + namespace: test-gslb-istio +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + run: unhealthy-app + sessionAffinity: None + type: ClusterIP diff --git a/deploy/test-apps/unhealthy-app.yaml b/deploy/test-apps/unhealthy-app.yaml index 03dcda7621..5e75f3417a 100644 --- a/deploy/test-apps/unhealthy-app.yaml +++ b/deploy/test-apps/unhealthy-app.yaml @@ -1,3 +1,4 @@ +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -19,3 +20,25 @@ spec: - image: nginx imagePullPolicy: Always name: unhealthy-app +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + run: unhealthy-app + name: unhealthy-app + namespace: test-gslb-istio +spec: + replicas: 0 + selector: + matchLabels: + run: unhealthy-app + template: + metadata: + labels: + run: unhealthy-app + spec: + containers: + - image: nginx + imagePullPolicy: Always + name: unhealthy-app diff --git a/docs/local.md b/docs/local.md index eb00b58c14..d51e331fab 100644 --- a/docs/local.md +++ b/docs/local.md @@ -137,7 +137,7 @@ make destroy-full-local-setup Both clusters have [podinfo](https://github.com/stefanprodan/podinfo) installed on the top, where each cluster has been tagged to serve a different region. In this demo we will hit podinfo by `wget -qO - roundrobin.cloud.example.com` and depending on the region, podinfo will return **us** or **eu**. In the current round robin implementation IP addresses are randomly picked. -See [Gslb manifest with round robin strategy](https://github.com/k8gb-io/k8gb/tree/master/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr.yaml) +See [Gslb manifest with round robin strategy](https://github.com/k8gb-io/k8gb/tree/master/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_roundrobin_ingress.yaml) Try to run the following command several times and watch the `message` field. ```sh @@ -167,7 +167,7 @@ As expected result you should see podinfo message changing Both clusters have [podinfo](https://github.com/stefanprodan/podinfo) installed on the top where each cluster has been tagged to serve a different region. In this demo we will hit podinfo by `wget -qO - failover.cloud.example.com` and depending on whether podinfo is running inside the cluster it returns only **eu** or **us**. -See [Gslb manifest with failover strategy](https://github.com/k8gb-io/k8gb/tree/master/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover.yaml) +See [Gslb manifest with failover strategy](https://github.com/k8gb-io/k8gb/tree/master/deploy/crds/k8gb.absa.oss_v1beta1_gslb_cr_failover_ingress.yaml) Switch GLSB to failover mode: ```sh diff --git a/go.mod b/go.mod index 84a5ecec2c..a81030413a 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( go.opentelemetry.io/otel/sdk v1.28.0 go.opentelemetry.io/otel/trace v1.28.0 go.uber.org/mock v0.4.0 + istio.io/client-go v1.22.2 k8s.io/api v0.30.2 k8s.io/apimachinery v0.30.2 k8s.io/client-go v0.30.2 @@ -82,6 +83,7 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + istio.io/api v1.22.2-0.20240619002433-8214ad1c7bd7 // indirect k8s.io/apiextensions-apiserver v0.30.1 // indirect k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20240423202451-8948a665c108 // indirect diff --git a/go.sum b/go.sum index 74f7c33146..aac9df7357 100644 --- a/go.sum +++ b/go.sum @@ -353,6 +353,10 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +istio.io/api v1.22.2-0.20240619002433-8214ad1c7bd7 h1:iSnlKJkc4UREFpBnaJLnZv2dtpg5DXUbk0BqTXpa3n0= +istio.io/api v1.22.2-0.20240619002433-8214ad1c7bd7/go.mod h1:S3l8LWqNYS9yT+d4bH+jqzH2lMencPkW7SKM1Cu9EyM= +istio.io/client-go v1.22.2 h1:BiE7itlXFTHpZwOv0t2aZQGga7oCox8lYOdaYbyWNEo= +istio.io/client-go v1.22.2/go.mod h1:Fxt0tVZLXQRKyrBv7uwm4zCZE0qayejG0bSwZy9K6Hg= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.30.2 h1:+ZhRj+28QT4UOH+BKznu4CBgPWgkXO7XAvMcMl0qKvI= k8s.io/api v0.30.2/go.mod h1:ULg5g9JvOev2dG0u2hig4Z7tQ2hHIuS+m8MNZ+X6EmI= diff --git a/k3d/test-gslb1.yaml b/k3d/test-gslb1.yaml index 502af8e718..3fc48deccd 100644 --- a/k3d/test-gslb1.yaml +++ b/k3d/test-gslb1.yaml @@ -11,6 +11,9 @@ ports: - port: 80:80 nodeFilters: - agent:0:direct + - port: 8080:8080 + nodeFilters: + - agent:0:direct - port: 443:443 nodeFilters: - agent:0:direct @@ -31,7 +34,7 @@ options: disableLoadbalancer: true k3s: extraArgs: - - arg: --disable=traefik,servicelb,metrics-server,local-storage + - arg: --disable=traefik,metrics-server,local-storage nodeFilters: - server:* registries: diff --git a/k3d/test-gslb2.yaml b/k3d/test-gslb2.yaml index db95b10136..8292a9c4a8 100644 --- a/k3d/test-gslb2.yaml +++ b/k3d/test-gslb2.yaml @@ -11,6 +11,9 @@ ports: - port: 81:80 nodeFilters: - agent:0:direct + - port: 8081:8080 + nodeFilters: + - agent:0:direct - port: 444:443 nodeFilters: - agent:0:direct @@ -28,7 +31,7 @@ options: disableLoadbalancer: true k3s: extraArgs: - - arg: --disable=traefik,servicelb,metrics-server,local-storage + - arg: --disable=traefik,metrics-server,local-storage nodeFilters: - server:* registries: diff --git a/k3d/test-gslb3.yaml b/k3d/test-gslb3.yaml index c19231e5b1..8e0308ab5d 100644 --- a/k3d/test-gslb3.yaml +++ b/k3d/test-gslb3.yaml @@ -11,6 +11,9 @@ ports: - port: 82:80 nodeFilters: - agent:0:direct + - port: 8082:8080 + nodeFilters: + - agent:0:direct - port: 445:443 nodeFilters: - agent:0:direct @@ -28,7 +31,7 @@ options: disableLoadbalancer: true k3s: extraArgs: - - arg: --disable=traefik,servicelb,metrics-server,local-storage + - arg: --disable=traefik,metrics-server,local-storage nodeFilters: - server:* registries: diff --git a/main.go b/main.go index a9657c3af9..2cb8a7e5c1 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,7 @@ import ( "github.com/k8gb-io/k8gb/controllers/providers/dns" "github.com/k8gb-io/k8gb/controllers/providers/metrics" "github.com/k8gb-io/k8gb/controllers/tracing" + istio "istio.io/client-go/pkg/apis/networking/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -49,8 +50,8 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(runtimescheme)) - utilruntime.Must(k8gbv1beta1.AddToScheme(runtimescheme)) + utilruntime.Must(istio.AddToScheme(runtimescheme)) // +kubebuilder:scaffold:scheme } diff --git a/terratest/examples/failover-istio-gateway.yaml b/terratest/examples/failover-istio-gateway.yaml new file mode 100644 index 0000000000..55763a0a63 --- /dev/null +++ b/terratest/examples/failover-istio-gateway.yaml @@ -0,0 +1,15 @@ +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: terratest-failover + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - terratest-failover.cloud.example.com + port: + name: http + number: 8080 + protocol: http diff --git a/terratest/examples/failover-istio-gslb.yaml b/terratest/examples/failover-istio-gslb.yaml new file mode 100644 index 0000000000..da3bbae8de --- /dev/null +++ b/terratest/examples/failover-istio-gslb.yaml @@ -0,0 +1,13 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: terratest-failover +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: terratest-failover + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" diff --git a/terratest/examples/failover-istio-virtualservice.yaml b/terratest/examples/failover-istio-virtualservice.yaml new file mode 100644 index 0000000000..7daed85014 --- /dev/null +++ b/terratest/examples/failover-istio-virtualservice.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: terratest-failover + labels: + app: terratest-failover +spec: + gateways: + - istio-ingress/terratest-failover + hosts: + - terratest-failover.cloud.example.com + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 diff --git a/terratest/examples/failover-playground-istio-gateway.yaml b/terratest/examples/failover-playground-istio-gateway.yaml new file mode 100644 index 0000000000..7faac5fa47 --- /dev/null +++ b/terratest/examples/failover-playground-istio-gateway.yaml @@ -0,0 +1,15 @@ +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: playground-failover + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - playground-failover.cloud.example.com + port: + name: http + number: 8080 + protocol: http diff --git a/terratest/examples/failover-playground-istio-gslb.yaml b/terratest/examples/failover-playground-istio-gslb.yaml new file mode 100644 index 0000000000..256a4b0241 --- /dev/null +++ b/terratest/examples/failover-playground-istio-gslb.yaml @@ -0,0 +1,13 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: playground-failover +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: playground-failover + strategy: + type: failover + dnsTtlSeconds: 5 + primaryGeoTag: "eu" diff --git a/terratest/examples/failover-playground-istio-virtualservice.yaml b/terratest/examples/failover-playground-istio-virtualservice.yaml new file mode 100644 index 0000000000..28e3072cfa --- /dev/null +++ b/terratest/examples/failover-playground-istio-virtualservice.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: playground-failover + labels: + app: playground-failover +spec: + gateways: + - istio-ingress/playground-failover + hosts: + - playground-failover.cloud.example.com + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 diff --git a/terratest/examples/failover-playground-ref-gslb.yaml b/terratest/examples/failover-playground-ref-gslb.yaml index fa77797c58..bc5c535112 100644 --- a/terratest/examples/failover-playground-ref-gslb.yaml +++ b/terratest/examples/failover-playground-ref-gslb.yaml @@ -1,12 +1,12 @@ apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: - name: test-gslb + name: playground-failover spec: resourceRef: ingress: matchLabels: - app: test-gslb + app: playground-failover strategy: type: failover dnsTtlSeconds: 5 diff --git a/terratest/examples/failover-playground-ref-ingress.yaml b/terratest/examples/failover-playground-ref-ingress.yaml index 509542e81e..355c7991c3 100644 --- a/terratest/examples/failover-playground-ref-ingress.yaml +++ b/terratest/examples/failover-playground-ref-ingress.yaml @@ -1,9 +1,9 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: test-gslb + name: playground-failover labels: - app: test-gslb + app: playground-failover spec: ingressClassName: nginx rules: diff --git a/terratest/examples/failover-playground.yaml b/terratest/examples/failover-playground.yaml index 031e77070c..dc3ed0eed5 100644 --- a/terratest/examples/failover-playground.yaml +++ b/terratest/examples/failover-playground.yaml @@ -1,7 +1,7 @@ apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: - name: test-gslb + name: playground-failover spec: ingress: ingressClassName: nginx diff --git a/terratest/examples/failover.yaml b/terratest/examples/failover.yaml index b6694e9b2e..fba6772cb7 100644 --- a/terratest/examples/failover.yaml +++ b/terratest/examples/failover.yaml @@ -9,13 +9,13 @@ spec: - host: terratest-failover.cloud.example.com http: paths: - - path: / - pathType: Prefix - backend: - service: - name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records - port: - name: http + - path: / + pathType: Prefix + backend: + service: + name: frontend-podinfo # Gslb should reflect Healthy status and create associated DNS records + port: + name: http strategy: type: failover dnsTtlSeconds: 5 diff --git a/terratest/examples/roundrobin-weight1-istio-gateway.yaml b/terratest/examples/roundrobin-weight1-istio-gateway.yaml new file mode 100644 index 0000000000..b1fa1212d0 --- /dev/null +++ b/terratest/examples/roundrobin-weight1-istio-gateway.yaml @@ -0,0 +1,15 @@ +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: test-gslb + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - terratest-roundrobin.cloud.example.com + port: + name: http + number: 8080 + protocol: http diff --git a/terratest/examples/roundrobin-weight1-istio-gslb.yaml b/terratest/examples/roundrobin-weight1-istio-gslb.yaml new file mode 100644 index 0000000000..8a104aee32 --- /dev/null +++ b/terratest/examples/roundrobin-weight1-istio-gslb.yaml @@ -0,0 +1,15 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: test-gslb +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: test-gslb + strategy: + type: roundRobin + dnsTtlSeconds: 5 + weight: + eu: 5 + us: 5 diff --git a/terratest/examples/roundrobin-weight1-istio-virtualservice.yaml b/terratest/examples/roundrobin-weight1-istio-virtualservice.yaml new file mode 100644 index 0000000000..71882df1ba --- /dev/null +++ b/terratest/examples/roundrobin-weight1-istio-virtualservice.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: test-gslb + labels: + app: test-gslb +spec: + gateways: + - istio-ingress/test-gslb + hosts: + - terratest-roundrobin.cloud.example.com + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 diff --git a/terratest/examples/roundrobin2-istio-gateway.yaml b/terratest/examples/roundrobin2-istio-gateway.yaml new file mode 100644 index 0000000000..a0d768ee1b --- /dev/null +++ b/terratest/examples/roundrobin2-istio-gateway.yaml @@ -0,0 +1,15 @@ +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: roundrobin-test-gslb + namespace: istio-ingress +spec: + selector: + app: istio-ingressgateway + servers: + - hosts: + - roundrobin-test.cloud.example.com + port: + name: http + number: 8080 + protocol: http diff --git a/terratest/examples/roundrobin2-istio-gslb.yaml b/terratest/examples/roundrobin2-istio-gslb.yaml new file mode 100644 index 0000000000..d452fe72d1 --- /dev/null +++ b/terratest/examples/roundrobin2-istio-gslb.yaml @@ -0,0 +1,12 @@ +apiVersion: k8gb.absa.oss/v1beta1 +kind: Gslb +metadata: + name: roundrobin-test-gslb +spec: + resourceRef: + istioVirtualService: + matchLabels: + app: roundrobin-test + strategy: + type: roundRobin + dnsTtlSeconds: 5 diff --git a/terratest/examples/roundrobin2-istio-virtualservice.yaml b/terratest/examples/roundrobin2-istio-virtualservice.yaml new file mode 100644 index 0000000000..4b04bc9312 --- /dev/null +++ b/terratest/examples/roundrobin2-istio-virtualservice.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: roundrobin-test-gslb + labels: + app: roundrobin-test +spec: + gateways: + - istio-ingress/roundrobin-test-gslb + hosts: + - roundrobin-test.cloud.example.com + http: + - route: + - destination: + host: frontend-podinfo + port: + number: 9898 diff --git a/terratest/test/k8gb_abstract_full_roundrobin_test.go b/terratest/test/k8gb_abstract_full_roundrobin_test.go index 33a12c127f..21bb8b26e9 100644 --- a/terratest/test/k8gb_abstract_full_roundrobin_test.go +++ b/terratest/test/k8gb_abstract_full_roundrobin_test.go @@ -26,33 +26,25 @@ import ( "github.com/stretchr/testify/require" ) -func abstractTestFullRoundRobin(t *testing.T, n int, testPrefix string, gslbPath string, ingressPath string) { - if n < 2 || n > 8 { - t.Logf("Use value of n that represents the number of clusters from interval [2,8]") +func abstractTestFullRoundRobin(t *testing.T, testPrefix string, workflows []*utils.Workflow) { + if len(workflows) < 2 || len(workflows) > 8 { + t.Logf("Number of cluster must be in the interval [2,8]") t.FailNow() } - t.Logf(fmt.Sprintf("Running TestFullRoundRobin for %d clusters", n)) + t.Logf(fmt.Sprintf("Running TestFullRoundRobin for %d clusters", len(workflows))) tags := []string{"eu", "us", "cz", "af", "ru", "ap", "uk", "ca"} var instances []*utils.Instance - const host = "roundrobin-test.cloud.example.com" - // start all the test apps on all the clusters - for i := 0; i < n; i += 1 { - workflow := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i). - WithGslb(gslbPath, host). - WithTestApp(tags[i]) - if ingressPath != "" { - workflow = workflow.WithIngress(ingressPath) - } - + for i, workflow := range workflows { + workflow = workflow.WithTestApp(tags[i]) instance, er := workflow.Start() require.NoError(t, er) instances = append(instances, instance) defer instance.Kill() } var err error - t.Run(fmt.Sprintf("%s round-robin on %d concurrent clusters with podinfo running", testPrefix, n), func(t *testing.T) { + t.Run(fmt.Sprintf("%s round-robin on %d concurrent clusters with podinfo running", testPrefix, len(workflows)), func(t *testing.T) { for _, ins := range instances { err = ins.WaitForAppIsRunning() require.NoError(t, err) @@ -64,7 +56,7 @@ func abstractTestFullRoundRobin(t *testing.T, n int, testPrefix string, gslbPath for _, ins := range instances { workingTargets = append(workingTargets, ins.GetLocalTargets()...) } - t.Run(fmt.Sprintf("%s all %d clusters should be interconnected", testPrefix, n), func(t *testing.T) { + t.Run(fmt.Sprintf("%s all %d clusters should be interconnected", testPrefix, len(workflows)), func(t *testing.T) { allShouldExpectTheseTargets(t, instances, workingTargets) }) diff --git a/terratest/test/k8gb_failover_playground_test.go b/terratest/test/k8gb_failover_playground_test.go index 032a061de1..9694c13a19 100644 --- a/terratest/test/k8gb_failover_playground_test.go +++ b/terratest/test/k8gb_failover_playground_test.go @@ -33,50 +33,29 @@ import ( // TestFailoverPlayground is equal to k8gb failover test running on local playground. // see: https://github.com/k8gb-io/k8gb/blob/master/docs/local.md#failover func TestFailoverPlayground(t *testing.T) { - tests := []struct { - name string - gslbPath string - ingressPath string - }{ - { - name: "embedded ingress", - gslbPath: "../examples/failover-playground.yaml", - ingressPath: "", - }, - { - name: "referenced ingress", - gslbPath: "../examples/failover-playground-ref-gslb.yaml", - ingressPath: "../examples/failover-playground-ref-ingress.yaml", - }, - } - - for _, test := range tests { - abstractTestFailoverPlayground(t, test.name, test.gslbPath, test.ingressPath) + for _, ingressType := range utils.IngressTypes { + const basePath = "../examples/failover-playground" + const host = "playground-failover.cloud.example.com" + + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053) + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054) + abstractTestFailoverPlayground(t, ingressType.String(), + workflowEU.Enrich(basePath, host, ingressType), + workflowUS.Enrich(basePath, host, ingressType), + ) } } -func abstractTestFailoverPlayground(t *testing.T, testPrefix string, gslbPath string, ingressPath string) { - const host = "playground-failover.cloud.example.com" - +func abstractTestFailoverPlayground(t *testing.T, testPrefix string, workflowEU, workflowUS *utils.Workflow) { const euGeoTag = "eu" const usGeoTag = "us" - workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). - WithGslb(gslbPath, host). - WithTestApp(euGeoTag) - if ingressPath != "" { - workflowEU = workflowEU.WithIngress(ingressPath) - } + workflowEU = workflowEU.WithTestApp(euGeoTag) instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). - WithGslb(gslbPath, host). - WithTestApp(usGeoTag) - if ingressPath != "" { - workflowUS = workflowUS.WithIngress(ingressPath) - } + workflowUS = workflowUS.WithTestApp(usGeoTag) instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() diff --git a/terratest/test/k8gb_full_failover_test.go b/terratest/test/k8gb_full_failover_test.go index dd065e4055..2c7837d522 100644 --- a/terratest/test/k8gb_full_failover_test.go +++ b/terratest/test/k8gb_full_failover_test.go @@ -30,47 +30,26 @@ import ( ) func TestFullFailover(t *testing.T) { - tests := []struct { - name string - gslbPath string - ingressPath string - }{ - { - name: "embedded ingress", - gslbPath: "../examples/failover.yaml", - ingressPath: "", - }, - { - name: "referenced ingress", - gslbPath: "../examples/failover-ref-gslb.yaml", - ingressPath: "../examples/failover-ref-ingress.yaml", - }, - } - - for _, test := range tests { - abstractTestFullFailover(t, test.name, test.gslbPath, test.ingressPath) + for _, ingressType := range utils.IngressTypes { + const basePath = "../examples/failover" + const host = "terratest-failover.cloud.example.com" + + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053) + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054) + abstractTestFullFailover(t, ingressType.String(), + workflowEU.Enrich(basePath, host, ingressType), + workflowUS.Enrich(basePath, host, ingressType), + ) } } -func abstractTestFullFailover(t *testing.T, testPrefix string, gslbPath string, ingressPath string) { - const host = "terratest-failover.cloud.example.com" - - workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). - WithGslb(gslbPath, host). - WithTestApp("eu") - if ingressPath != "" { - workflowEU = workflowEU.WithIngress(ingressPath) - } +func abstractTestFullFailover(t *testing.T, testPrefix string, workflowEU, workflowUS *utils.Workflow) { + workflowEU = workflowEU.WithTestApp("eu") instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). - WithGslb(gslbPath, host). - WithTestApp("us") - if ingressPath != "" { - workflowUS = workflowUS.WithIngress(ingressPath) - } + workflowUS = workflowUS.WithTestApp("us") instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() diff --git a/terratest/test/k8gb_full_roundrobin_test.go b/terratest/test/k8gb_full_roundrobin_test.go index bbb5d60f36..78236f68f2 100644 --- a/terratest/test/k8gb_full_roundrobin_test.go +++ b/terratest/test/k8gb_full_roundrobin_test.go @@ -22,28 +22,23 @@ Generated by GoLic, for more details see: https://github.com/AbsaOSS/golic */ import ( + "fmt" + "k8gbterratest/utils" "testing" ) func TestFullRoundRobin(t *testing.T) { - tests := []struct { - name string - gslbPath string - ingressPath string - }{ - { - name: "embedded ingress", - gslbPath: "../examples/roundrobin2.yaml", - ingressPath: "", - }, - { - name: "referenced ingress", - gslbPath: "../examples/roundrobin2-ref-gslb.yaml", - ingressPath: "../examples/roundrobin2-ref-ingress.yaml", - }, - } - - for test := range tests { - abstractTestFullRoundRobin(t, settings.ClusterNumber, test.name, test.gslbPath, test.ingressPath) + for _, ingressType := range utils.IngressTypes { + const basePath = "../examples/roundrobin2" + const host = "roundrobin-test.cloud.example.com" + + workflows := []*utils.Workflow{} + for i := 0; i < settings.ClustersNumber; i += 1 { + workflow := utils.NewWorkflow(t, fmt.Sprintf("k3d-test-gslb%d", i+1), 5053+i) + workflow.Enrich(basePath, host, ingressType) + workflows = append(workflows, workflow) + } + + abstractTestFullRoundRobin(t, ingressType.String(), workflows) } } diff --git a/terratest/test/k8gb_weight_test.go b/terratest/test/k8gb_weight_test.go index 1988114d62..b881c3cc0a 100644 --- a/terratest/test/k8gb_weight_test.go +++ b/terratest/test/k8gb_weight_test.go @@ -27,50 +27,30 @@ import ( ) func TestWeightsExistsInLocalDNSEndpoint(t *testing.T) { - tests := []struct { - gslbPath string - ingressPath string - name string - }{ - { - name: "with gslb only", - gslbPath: "../examples/roundrobin-weight1.yaml", - ingressPath: "", - }, - { - name: "with ref", - gslbPath: "../examples/roundrobin-weight1-ref-gslb.yaml", - ingressPath: "../examples/roundrobin-weight1-ref-ingress.yaml", - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - abstractTestWeightsExistsInLocalDNSEndpoint(t, test.gslbPath, test.ingressPath) + for _, ingressType := range utils.IngressTypes { + const basePath = "../examples/roundrobin-weight1" + const host = "terratest-roundrobin.cloud.example.com" + + workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053) + workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054) + t.Run(ingressType.String(), func(t *testing.T) { + abstractTestWeightsExistsInLocalDNSEndpoint(t, host, + workflowEU.Enrich(basePath, host, ingressType), + workflowUS.Enrich(basePath, host, ingressType), + ) }) } } -func abstractTestWeightsExistsInLocalDNSEndpoint(t *testing.T, gslbPath string, ingressPath string) { - const host = "terratest-roundrobin.cloud.example.com" +func abstractTestWeightsExistsInLocalDNSEndpoint(t *testing.T, host string, workflowEU, workflowUS *utils.Workflow) { const endpointDNSNameEU = "gslb-ns-eu-cloud.example.com" const endpointDNSNameUS = "gslb-ns-us-cloud.example.com" - workflowEU := utils.NewWorkflow(t, "k3d-test-gslb1", 5053). - WithGslb(gslbPath, host). - WithTestApp("eu") - if ingressPath != "" { - workflowEU.WithIngress(ingressPath) - } + workflowEU = workflowEU.WithTestApp("eu") instanceEU, err := workflowEU.Start() require.NoError(t, err) defer instanceEU.Kill() - workflowUS := utils.NewWorkflow(t, "k3d-test-gslb2", 5054). - WithGslb(gslbPath, host). - WithTestApp("us") - if ingressPath != "" { - workflowUS.WithIngress(ingressPath) - } + workflowUS = workflowUS.WithTestApp("us") instanceUS, err := workflowUS.Start() require.NoError(t, err) defer instanceUS.Kill() diff --git a/terratest/utils/extensions.go b/terratest/utils/extensions.go index b3f7d25dbb..7510963b52 100644 --- a/terratest/utils/extensions.go +++ b/terratest/utils/extensions.go @@ -44,16 +44,21 @@ import ( ) type Workflow struct { - error error - namespace string - cluster string - k8sOptions *k8s.KubectlOptions - t *testing.T - settings struct { - ingressResourcePath string - gslbResourcePath string - ingressName string - digUsingUDP bool + error error + namespace string + cluster string + k8sOptions *k8s.KubectlOptions + k8sOptionsIstioIngress *k8s.KubectlOptions + t *testing.T + settings struct { + ingressType IngressType + istioVirtualServiceResourcePath string + istioGatewayResourcePath string + ingressResourcePath string + gslbResourcePath string + ingressName string + ingressPort int + digUsingUDP bool } state struct { namespaceCreated bool @@ -104,6 +109,40 @@ type InstanceStatus struct { EndpointGlobalTargets string `json:"ep1-dns-targets"` } +type IngressType int + +const ( + IngressEmbedded IngressType = iota + IngressReferenced + IngressIstio +) + +const ( + istioIngressNamespace = "istio-ingress" + istioIngressLoadBalancerService = "istio-ingressgateway" + kubernetesIngressPort = 80 + istioIngressPort = 8080 +) + +var IngressTypes = []IngressType{ + IngressEmbedded, + IngressReferenced, + IngressIstio, +} + +func (it IngressType) String() string { + switch it { + case IngressEmbedded: + return "ingress embedded" + case IngressReferenced: + return "ingress referenced" + case IngressIstio: + return "ingress istio" + default: + return "ingress unknown" + } +} + func NewWorkflow(t *testing.T, cluster string, port int) *Workflow { var err error if cluster == "" { @@ -116,12 +155,40 @@ func NewWorkflow(t *testing.T, cluster string, port int) *Workflow { w.cluster = cluster w.namespace = fmt.Sprintf("k8gb-test-%s", strings.ToLower(random.UniqueId())) w.k8sOptions = k8s.NewKubectlOptions(cluster, "", w.namespace) + w.k8sOptionsIstioIngress = k8s.NewKubectlOptions(cluster, "", istioIngressNamespace) w.t = t w.state.gslb.port = port w.error = err return w } +// Enrich enriches the workflow with the relevant resources depending on the ingressType +func (w *Workflow) Enrich(basePath string, host string, ingressType IngressType) *Workflow { + gslbPath := "" + ingressPort := kubernetesIngressPort + + switch ingressType { + case IngressEmbedded: + gslbPath = basePath + ".yaml" + case IngressReferenced: + gslbPath = basePath + "-ref-gslb.yaml" + ingressPath := basePath + "-ref-ingress.yaml" + w = w.WithIngress(ingressPath) + case IngressIstio: + gslbPath = basePath + "-istio-gslb.yaml" + virtualServicePath := basePath + "-istio-virtualservice.yaml" + gatewayPath := basePath + "-istio-gateway.yaml" + w = w.WithIstio(virtualServicePath, gatewayPath) + ingressPort = istioIngressPort + } + + w = w.WithGslb(gslbPath, host) + w.settings.ingressType = ingressType + w.settings.ingressPort = ingressPort + + return w +} + func (w *Workflow) WithIngress(path string) *Workflow { if path == "" { w.error = fmt.Errorf("empty ingress resource path") @@ -138,6 +205,27 @@ func (w *Workflow) WithIngress(path string) *Workflow { return w } +func (w *Workflow) WithIstio(virtualServicePath, gatewayPath string) *Workflow { + if virtualServicePath == "" { + w.error = fmt.Errorf("empty istio virtual service resource path") + } + if gatewayPath == "" { + w.error = fmt.Errorf("empty istio gateway resource path") + } + + var err error + w.settings.istioVirtualServiceResourcePath, err = filepath.Abs(virtualServicePath) + if err != nil { + w.error = fmt.Errorf("reading %s; %s", virtualServicePath, err) + } + w.settings.istioGatewayResourcePath, err = filepath.Abs(gatewayPath) + if err != nil { + w.error = fmt.Errorf("reading %s; %s", gatewayPath, err) + } + + return w +} + // WithGslb // TODO: consider taking host dynamically func (w *Workflow) WithGslb(path, host string) *Workflow { @@ -179,7 +267,14 @@ func (w *Workflow) Start() (*Instance, error) { // namespace w.t.Logf("Create namespace %s", w.namespace) - k8s.CreateNamespace(w.t, w.k8sOptions, w.namespace) + istioInjection := "disabled" + if w.settings.ingressType == IngressIstio { + istioInjection = "enabled" + } + k8s.CreateNamespaceWithMetadata(w.t, w.k8sOptions, metav1.ObjectMeta{ + Name: w.namespace, + Labels: map[string]string{"istio-injection": istioInjection}, + }) w.state.namespaceCreated = true // app @@ -217,19 +312,27 @@ func (w *Workflow) Start() (*Instance, error) { // gslb if w.settings.gslbResourcePath != "" { - if w.settings.ingressResourcePath == "" { + switch w.settings.ingressType { + case IngressEmbedded: w.t.Logf("Create ingress %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.state.gslb.name, 100, 5*time.Second) ingress := k8s.GetIngress(w.t, w.k8sOptions, w.state.gslb.name) require.Equal(w.t, ingress.Name, w.state.gslb.name) w.settings.ingressName = w.state.gslb.name - } else { + case IngressReferenced: w.t.Logf("Create ingress %s from %s", w.settings.ingressName, w.settings.ingressResourcePath) k8s.KubectlApply(w.t, w.k8sOptions, w.settings.ingressResourcePath) k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.settings.ingressName, 100, 5*time.Second) w.t.Logf("Create gslb %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) + case IngressIstio: + w.t.Logf("Create istio virtual service from %s", w.settings.istioVirtualServiceResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.istioVirtualServiceResourcePath) + w.t.Logf("Create istio gateway from %s", w.settings.istioGatewayResourcePath) + k8s.KubectlApply(w.t, w.k8sOptionsIstioIngress, w.settings.istioGatewayResourcePath) + w.t.Logf("Create gslb %s from %s", w.state.gslb.name, w.settings.gslbResourcePath) + k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath) } } @@ -261,6 +364,9 @@ func (i *Instance) Kill() { if i.w.state.namespaceCreated { k8s.DeleteNamespace(i.w.t, i.w.k8sOptions, i.w.namespace) } + if i.w.settings.istioGatewayResourcePath != "" { + k8s.KubectlDelete(i.w.t, i.w.k8sOptionsIstioIngress, i.w.settings.istioGatewayResourcePath) + } } func (i *Instance) ReapplyIngress(path string) { @@ -293,9 +399,17 @@ func (i *Instance) GetCoreDNSIP() string { func (i *Instance) GetIngressIPs() []string { var ingressIPs []string - ingress := k8s.GetIngress(i.w.t, i.w.k8sOptions, i.w.settings.ingressName) - for _, ip := range ingress.Status.LoadBalancer.Ingress { - ingressIPs = append(ingressIPs, ip.IP) + switch i.w.settings.ingressType { + case IngressIstio: + lbService := k8s.GetService(i.w.t, i.w.k8sOptionsIstioIngress, istioIngressLoadBalancerService) + for _, ip := range lbService.Status.LoadBalancer.Ingress { + ingressIPs = append(ingressIPs, ip.IP) + } + default: + ingress := k8s.GetIngress(i.w.t, i.w.k8sOptions, i.w.settings.ingressName) + for _, ip := range ingress.Status.LoadBalancer.Ingress { + ingressIPs = append(ingressIPs, ip.IP) + } } return ingressIPs } @@ -487,7 +601,7 @@ func (i *Instance) HitTestApp() (result *TestAppResult) { var err error result = new(TestAppResult) coreDNSIP := i.GetCoreDNSIP() - command := []string{"sh", "-c", fmt.Sprintf("wget -qO - %s", i.w.state.gslb.host)} + command := []string{"sh", "-c", fmt.Sprintf("wget -qO - %s:%d", i.w.state.gslb.host, i.w.settings.ingressPort)} for t := 0; t < 60; t++ { result.Body, err = RunBusyBoxCommand(i.w.t, i.w.k8sOptions, coreDNSIP, command) if strings.Contains(result.Body, "503") { @@ -570,7 +684,7 @@ func waitForLocalGSLBNew(t *testing.T, host string, port int, expectedResult []s t, "Wait for failover to happen and coredns to pickup new values...", DefaultRetries, - time.Second*1, + time.Second*3, func() ([]string, error) { return dns.Dig("localhost:"+strconv.Itoa(port), host, isUdp) }, expectedResult) } diff --git a/terratest/utils/utils.go b/terratest/utils/utils.go index d5d739c277..6d072775c7 100644 --- a/terratest/utils/utils.go +++ b/terratest/utils/utils.go @@ -231,7 +231,7 @@ func WaitForLocalGSLB(t *testing.T, dnsServer string, dnsPort int, settings Test t, "Wait for failover to happen and coredns to pickup new values...", 300, - time.Second*1, + time.Second*3, func() ([]string, error) { return Dig(t, dnsServer, dnsPort, host+settings.DNSZone, additionalArgs...) }, expectedResult) } @@ -261,7 +261,7 @@ func RunBusyBoxCommand(t *testing.T, options *k8s.KubectlOptions, dns string, co dnsOverride := fmt.Sprintf("{\"spec\":{\"dnsConfig\":{\"nameservers\":[\"%s\"]},\"dnsPolicy\": \"None\"}}", dns) args := []string{} kubectlCtx := []string{"--context", options.ContextName, "-n", options.Namespace} - containerArgs := []string{"run", "-i", "--rm", "busybox", "--restart", "Never", "--image", "busybox"} + containerArgs := []string{"run", "-i", "--rm", "busybox", "--restart", "Never", "--image", "busybox", "--labels", "sidecar.istio.io/inject=false"} containerDNS := []string{"--overrides", dnsOverride} appArgs := append([]string{"--"}, command...) args = append(kubectlCtx, containerArgs...)