Integration Test suite + kind workflow to add them to ci

.github/resources/dspa-lite/dspa.yaml

@@ -0,0 +1,52 @@
+apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1
+kind: DataSciencePipelinesApplication
+metadata:
+  name: test-dspa
+spec:
+  apiServer:
+    enableOauth: false
+    resources:
+      limits:
+        cpu: 20m
+        memory: 500Mi
+      requests:
+        cpu: 20m
+        memory: 100m
+  scheduledWorkflow:
+    resources:
+      limits:
+        cpu: 20m
+        memory: 500Mi
+      requests:
+        cpu: 20m
+        memory: 100m
+  persistenceAgent:
+    resources:
+      limits:
+        cpu: 20m
+        memory: 500Mi
+      requests:
+        cpu: 20m
+        memory: 100Mi
+  database:
+    mariaDB:
+      image: quay.io/centos7/mariadb-103-centos7:ea07c0dade9571d78a272b453fd2dea92077dc7f
+      pvcSize: 500Mi
+      resources:
+        limits:
+          cpu: 20m
+          memory: 500Mi
+        requests:
+          cpu: 20m
+          memory: 100m
+  objectStorage:
+    minio:
+      image: 'quay.io/opendatahub/minio:RELEASE.2019-08-14T20-37-41Z-license-compliance'
+      pvcSize: 500Mi
+      resources:
+        limits:
+          cpu: 20m
+          memory: 500Mi
+        requests:
+          cpu: 20m
+          memory: 100m

.github/resources/tekton-lite/anyuid-scc.yaml

@@ -0,0 +1,43 @@
+allowHostDirVolumePlugin: false
+allowHostIPC: false
+allowHostNetwork: false
+allowHostPID: false
+allowHostPorts: false
+allowPrivilegeEscalation: true
+allowPrivilegedContainer: false
+allowedCapabilities: null
+apiVersion: security.openshift.io/v1
+defaultAddCapabilities: null
+fsGroup:
+  type: MustRunAs
+groups: []
+kind: SecurityContextConstraints
+metadata:
+  name: tekton-anyuid
+priority: 10
+readOnlyRootFilesystem: false
+requiredDropCapabilities:
+  - KILL
+  - MKNOD
+  - SETUID
+  - SETGID
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: MustRunAs
+supplementalGroups:
+  type: RunAsAny
+users:
+  - system:serviceaccount:tekton-pipelines:tekton-pipelines-controller
+  - system:serviceaccount:tekton-pipelines:tekton-pipelines-webhook
+volumes:
+  - configMap
+  - csi
+  - downwardAPI
+  - emptyDir
+  - ephemeral
+  - persistentVolumeClaim
+  - projected
+  - secret
+seccompProfiles:
+  - runtime/default

.github/resources/tekton-lite/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://github.com/opendatahub-io/data-science-pipelines//manifests/kustomize/third-party/tekton/installs/cluster/?ref\=master
+  - anyuid-scc.yaml
+patchesStrategicMerge:
+  - patches/deployment_tekton-pipelines-controller.yaml
+  - patches/deployment_tekton-pipelines-remote-resolvers.yaml
+  - patches/deployment_tekton-pipelines-webhook.yaml
+  - patches/horizontalpodautoscaler_tekton-pipelines-webhook.yaml

.github/resources/tekton-lite/patches/deployment_tekton-pipelines-controller.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tekton-pipelines-controller
+  namespace: tekton-pipelines
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: tekton-pipelines-controller
+        resources:
+          limits:
+            cpu: 100m
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi

.github/resources/tekton-lite/patches/deployment_tekton-pipelines-remote-resolvers.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tekton-pipelines-remote-resolvers
+  namespace: tekton-pipelines-resolvers
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: controller
+        resources:
+          limits:
+            cpu: 100m
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi

.github/resources/tekton-lite/patches/deployment_tekton-pipelines-webhook.yaml

@@ -0,0 +1,18 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tekton-pipelines-webhook
+  namespace: tekton-pipelines
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+      - name: webhook
+        resources:
+          limits:
+            cpu: 100m
+            memory: 500Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi

.github/resources/tekton-lite/patches/horizontalpodautoscaler_tekton-pipelines-webhook.yaml

@@ -0,0 +1,27 @@
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  labels:
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/name: webhook
+    app.kubernetes.io/part-of: tekton-pipelines
+    app.kubernetes.io/version: v0.41.0
+    pipeline.tekton.dev/release: v0.41.0
+    version: v0.41.0
+  name: tekton-pipelines-webhook
+  namespace: tekton-pipelines
+spec:
+  maxReplicas: 1
+  metrics:
+  - resource:
+      name: cpu
+      target:
+        averageUtilization: 100
+        type: Utilization
+    type: Resource
+  minReplicas: 1
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: tekton-pipelines-webhook

.github/workflows/kind-integration.yml

@@ -0,0 +1,87 @@
+name: KinD DSPO/DSP Integration Tests
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - controllers/**
+      - api/**
+      - config/**
+      - tests/**
+    types:
+      - opened
+      - reopened
+      - closed
+      - synchronize
+
+concurrency:
+  group: ${{ github.head_ref }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  IMAGE_REPO_DSPO: data-science-pipelines-operator
+  DSPA_NAMESPACE: test-dspa
+  DSPA_NAME: test-dspa
+  DSPA_DEPLOY_WAIT_TIMEOUT: 300
+  INTEGRATION_TESTS_DIR: ${{ github.workspace }}/tests
+jobs:
+  dspo-tests:
+    runs-on: ubuntu-20.04
+    timeout-minutes: 60
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go
+        uses: actions/setup-go@v1
+        with:
+          go-version: '1.19.x'
+        id: go
+
+      - name: Setup and start KinD cluster
+        uses: ./.github/actions/kind
+
+      # TODO: Consolidate testing CRDS (2 locations)
+      - name: Apply OCP CRDs
+        env:
+          RESOURCES_DIR: ${{ github.workspace }}/.github/resources
+          CONFIG_DIR: ${{ github.workspace }}/config
+        run: |
+          kubectl apply -f ${{ env.RESOURCES_DIR }}/crds
+          kubectl apply -f ${{ env.CONFIG_DIR }}/crd/external/route.openshift.io_routes.yaml
+
+      - name: Build image
+        env:
+          DSPO_IMAGE: ${{env.REGISTRY_ADDRESS}}/data-science-pipelines-operator
+        run: |
+          make podman-build -e IMG="${DSPO_IMAGE}"
+
+      - name: Deploy DSPO
+        env:
+          DSPO_IMAGE: ${{env.REGISTRY_ADDRESS}}/data-science-pipelines-operator
+        run: |
+          kubectl create namespace opendatahub
+          make podman-push -e IMG="${DSPO_IMAGE}"
+          make deploy-kind -e IMG="${DSPO_IMAGE}"
+
+      - name: Wait for DSPO deployment
+        run: |
+          kubectl wait -n opendatahub --timeout=60s --for=condition=Available=true deployment data-science-pipelines-operator-controller-manager
+
+      - name: Create DSPA Namespace
+        run: |
+          kubectl create namespace ${{ env.DSPA_NAMESPACE }}
+
+      - name: Run tests
+        working-directory: ${{ github.workspace }}
+        env:
+          NAMESPACE: ${{ env.DSPA_NAMESPACE }}
+          DSPA_NAME: ${{ env.DSPA_NAME }}
+        run: |
+          make integrationtest K8SAPISERVERHOST=$(oc whoami --show-server) DSPANAMESPACE=${NAMESPACE}
+
+      - name: Clean up
+        run: |
+          make undeploy-kind

Makefile

@@ -53,6 +53,12 @@ ENVTEST_K8S_VERSION = 1.25.0
 # Namespace to deploy the operator
 OPERATOR_NS ?= opendatahub
 
+# Integration Test ENVvars
+KUBECONFIGPATH ?= $(HOME)/.kube/config
+K8SAPISERVERHOST ?= http://localhost:6443
+DSPANAMESPACE ?= default
+DSPAPATH ?= resources/dspa-lite.yaml
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -115,6 +121,11 @@ unittest: manifests generate fmt vet envtest ## Run tests.
 functest: manifests generate fmt vet envtest ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./... --tags=test_functional -coverprofile cover.out
 
+.PHONY: integrationtest
+integrationtest: ## Run integration tests
+	cd tests && \
+	go run github.com/onsi/ginkgo/v2/ginkgo --tags=test_integration -- -kubeconfig=${KUBECONFIGPATH} -k8sApiServerHost=${K8SAPISERVERHOST} -DSPANamespace=${DSPANAMESPACE} -DSPAPath=${DSPAPATH} -ginkgo.v
+
 ##@ Build
 
 .PHONY: build
@@ -126,7 +137,7 @@ run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
 .PHONY: podman-build
-podman-build: test ## Build container image with the manager.
+podman-build: ## Build container image with the manager.
 	podman build -t ${IMG} .
 
 .PHONY: podman-push
@@ -154,11 +165,24 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 		&& $(KUSTOMIZE) edit set namespace ${OPERATOR_NS}
 	$(KUSTOMIZE) build config/overlays/make-deploy | kubectl apply -f -
 
+.PHONY: deploy-kind
+deploy-kind:
+	cd config/overlays/kind-tests \
+		&& kustomize edit set image controller=${IMG} \
+		&& kustomize edit set namespace ${OPERATOR_NS}
+	kustomize build config/overlays/kind-tests | kubectl apply -f -
+
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	cd config/overlays/make-deploy && $(KUSTOMIZE) edit set namespace ${OPERATOR_NS}
 	$(KUSTOMIZE) build config/overlays/make-deploy | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
 
+.PHONY: undeploy-kind
+undeploy-kind: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	cd config/overlays/kind-tests \
+		&& kustomize edit set namespace ${OPERATOR_NS}
+	kustomize build config/overlays/kind-tests | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to

config/crd/external/monitoring.coreos.com_servicemonitors.yaml

@@ -1,3 +1,5 @@
+# This CRD is used for func/kind tests
+---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:

config/crd/external/route.openshift.io_routes.yaml

@@ -1,3 +1,4 @@
+# This CRD is used for func/kind tests
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition

config/manager/manager.yaml

@@ -34,6 +34,7 @@ spec:
         - /home/config
         image: $(IMAGES_DSPO)
         name: manager
+        imagePullPolicy: Always
         env:
           # Env vars are prioritized over --config
           - name: IMAGES_APISERVER

config/overlays/kind-tests/img_patch.yaml

@@ -0,0 +1,11 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: data-science-pipelines-operator-controller-manager
+  namespace: opendatahub
+spec:
+  template:
+    spec:
+      containers:
+        - image: controller
+          name: manager

config/overlays/kind-tests/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: opendatahub
+resources:
+  - ../../base
+patchesStrategicMerge:
+  - img_patch.yaml
+  - res_patch.yaml
+  - user_patch.yaml
+images:
+  - name: controller
+    newName: quay.io/opendatahub/data-science-pipelines-operator
+    newTag: main