feat!: Reimplement service with go-sdk (#351)

* chore: Remove useLocalFileSystem implementation Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Implement main and event handler in go-sdk Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Add go-sdk environment variables + Update code Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Start reimplementing unit tests Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Use go-utils APIV2 Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Add PUBSUB topic + Code quality changes Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Change uniform subscription default value Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Fix / escaping in GetAllKeptnResources Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Add logging to tests Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Add health endpoints in Helm file + Fix jobconfig integration tests Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Add mountFile unit tests + PR comments Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Remove log statements Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Convert eventhandler tests to go-sdk Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Fix JobConfigReader creation in event handler Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> * Add function documentation Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com> Signed-off-by: TannerGabriel <gabrieltanner.code@gmail.com>
keptn-contrib · Aug 31, 2022 · 91c9409 · 91c9409
1 parent f772d0c
commit 91c9409
Show file tree

Hide file tree

Showing 29 changed files with 1,003 additions and 1,292 deletions.
diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
@@ -32,7 +32,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        keptn-version: ["0.16.1", "0.17.0", "${{ needs.fetch_latest_keptn_versions.outputs.LATEST_KEPTN_RELEASE }}", "${{ needs.fetch_latest_keptn_versions.outputs.LATEST_KEPTN_PRERELEASE }}"] # https://github.com/keptn/keptn/releases
+        keptn-version: ["0.18.1", "${{ needs.fetch_latest_keptn_versions.outputs.LATEST_KEPTN_RELEASE }}", "${{ needs.fetch_latest_keptn_versions.outputs.LATEST_KEPTN_PRERELEASE }}"] # https://github.com/keptn/keptn/releases
         network-policy: [true, false]
         job-network-policy: [true, false]
     env:
@@ -42,6 +42,7 @@ jobs:
       BRANCH: ${{ github.head_ref || github.ref_name }}
       JES_E2E_TEST: true
       JES_NAMESPACE: keptn-jes-e2e
+      GITEA_PROVISIONER_VERSION: "0.1.1"
       GITEA_ADMIN_USERNAME: GiteaAdmin
       GITEA_NAMESPACE: gitea
       KUBECONFIG: /etc/rancher/k3s/k3s.yaml
@@ -124,7 +125,7 @@ jobs:
           GITEA_ADMIN_PASSWORD: ${{ steps.gitea_credentials.outputs.GITEA_ADMIN_PASSWORD }}
           GITEA_ENDPOINT: ${{ steps.gitea.outputs.GITEA_ENDPOINT }}
         run: |
-          helm install keptn-gitea-provisioner-service https://github.com/keptn-sandbox/keptn-gitea-provisioner-service/releases/download/0.1.0/keptn-gitea-provisioner-service-0.1.0.tgz \
+          helm install keptn-gitea-provisioner-service https://github.com/keptn-sandbox/keptn-gitea-provisioner-service/releases/download/${GITEA_PROVISIONER_VERSION}/keptn-gitea-provisioner-service-${GITEA_PROVISIONER_VERSION}.tgz \
             --set gitea.endpoint=${GITEA_ENDPOINT} \
             --set gitea.admin.create=true \
             --set gitea.admin.username=${GITEA_ADMIN_USERNAME} \

diff --git a/README.md b/README.md
@@ -35,7 +35,7 @@ running in the Keptn ecosystem:
 |    0.16.x     |                               keptncontrib/job-executor-service:0.2.3                                |       v2       |
 |    0.16.x     |                               keptncontrib/job-executor-service:0.2.4                                |       v2       |
 |    0.17.x     |                               keptncontrib/job-executor-service:0.2.5                                |       v2       |
-
+|    0.18.x     |                               keptncontrib/job-executor-service:0.3.0                                |       v2       |
 
 **Please Note**:
 - Newer Keptn versions might be compatible, but compatibility has not been verified at the time of the release.
@@ -91,7 +91,7 @@ keptn add-resource --project jes-example --service hello --stage production --re
 
 Trigger the `example-seq` sequence. 
 ```shell
-keptn trigger sequence --sequence example-seq --project jes-example --service hello --stage production
+keptn trigger sequence example-seq --project jes-example --service hello --stage production
 ```
 
 Have a look at the sequences for the project in Keptn bridge, it should look similar to the screenshot below:

diff --git a/chart/templates/configmap.yaml b/chart/templates/configmap.yaml
@@ -10,6 +10,7 @@ data:
   default_resource_requests_cpu: "50m"
   default_resource_requests_memory: "128Mi"
   keptn_api_endpoint: {{ include "job-executor-service.remote-control-plane.endpoint" . }}
+  configuration_service:   "{{ include "job-executor-service.remote-control-plane.endpoint" . }}/resource-service"
   default_job_service_account: "{{ include "job-executor-service.jobConfig.serviceAccountName" . }}"
   allow_privileged_jobs: "{{ .Values.jobConfig.allowPrivilegedJobs | default "false" }}"
   additional_job_labels: |

diff --git a/chart/templates/deployment.yaml b/chart/templates/deployment.yaml
@@ -39,6 +39,14 @@ spec:
             - name: http
               containerPort: 8080
               protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: 8080
+          readinessProbe:
+            httpGet:
+              path: /health
+              port: 8080
           volumeMounts:
             - mountPath: "/config"
               name: job-executor-settings
@@ -49,8 +57,8 @@ spec:
               secretKeyRef:
                 name: job-service-keptn-secrets
                 key: token
-          - name: CONFIGURATION_SERVICE
-            value: "http://localhost:8081/configuration-service"
+          - name: PUBSUB_TOPIC
+            value: {{ ((.Values).subscription).pubsubTopic | default "sh.keptn.>" }}
           - name: env
             value: 'production'
           - name: JOB_NAMESPACE
@@ -100,101 +108,76 @@ spec:
               configMapKeyRef:
                 name: job-service-config
                 key: task_deadline_seconds
+          - name: KEPTN_API_ENDPOINT
+            valueFrom:
+              configMapKeyRef:
+                name: job-service-config
+                key: keptn_api_endpoint
+          - name: CONFIGURATION_SERVICE
+            valueFrom:
+              configMapKeyRef:
+                name: job-service-config
+                key: configuration_service
+          - name: K8S_DEPLOYMENT_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: 'metadata.labels[''app.kubernetes.io/name'']'
+          - name: K8S_DEPLOYMENT_VERSION
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: 'metadata.labels[''app.kubernetes.io/version'']'
+          - name: K8S_DEPLOYMENT_COMPONENT
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: 'metadata.labels[''app.kubernetes.io/component'']'
+          - name: K8S_NAMESPACE
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.namespace
+          - name: K8S_NODE_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: spec.nodeName
+          - name: K8S_POD_NAME
+            valueFrom:
+              fieldRef:
+                apiVersion: v1
+                fieldPath: metadata.name
           - name: FULL_DEPLOYMENT_NAME
             value: {{ include "job-executor-service.fullname" . }}
-          resources:
-            {{- toYaml .Values.resources | nindent 12 }}
-        - name: distributor
-          image: "{{ .Values.distributor.image.repository }}:{{ .Values.distributor.image.tag | default .Chart.AppVersion }}"
-          securityContext:
-            {{- toYaml .Values.distributor.securityContext | nindent 12 }}
-          imagePullPolicy: Always
-          ports:
-            - containerPort: 8080
-          resources:
-            requests:
-              memory: "16Mi"
-              cpu: "25m"
-            limits:
-              memory: "32Mi"
-              cpu: "100m"
-          env:
-            - name: PUBSUB_TOPIC
-              value: "{{ .Values.remoteControlPlane.topicSubscription }}"
-            - name: PUBSUB_RECIPIENT
-              value: '127.0.0.1'
-            - name: STAGE_FILTER
-              value: "{{ .Values.distributor.stageFilter }}"
-            - name: PROJECT_FILTER
-              value: "{{ .Values.distributor.projectFilter }}"
-            - name: SERVICE_FILTER
-              value: "{{ .Values.distributor.serviceFilter }}"
-            - name: DISTRIBUTOR_VERSION
-              value: {{ .Values.distributor.image.tag | default .Chart.AppVersion }}
-            - name: VERSION
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.labels['app.kubernetes.io/version']
-            - name: K8S_DEPLOYMENT_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.labels['app.kubernetes.io/name']
-            - name: K8S_POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: K8S_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: K8S_NODE_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: KEPTN_API_ENDPOINT
-              valueFrom:
-                configMapKeyRef:
+          - name: HTTP_SSL_VERIFY
+            value: "{{ .Values.remoteControlPlane.api.apiValidateTls | default "true" }}"
+          {{- if eq ((.Values.remoteControlPlane).api.authMode) "oauth" }}
+          - name: OAUTH_CLIENT_ID
+            valueFrom:
+              configMapKeyRef:
                   name: job-service-config
-                  key: keptn_api_endpoint
-            {{- if eq ((.Values.remoteControlPlane).api.authMode) "token" }}
-            - name: KEPTN_API_TOKEN
-              valueFrom:
-                secretKeyRef:
+                  key: oauth_client_id
+          - name: OAUTH_CLIENT_SECRET
+            valueFrom:
+              secretKeyRef:
                   name: job-service-keptn-secrets
-                  key: token
-            {{- end }}
-            - name: HTTP_SSL_VERIFY
-              value: "{{ .Values.remoteControlPlane.api.apiValidateTls | default "true" }}"
-            {{- if (((.Values.distributor).config).queueGroup).enabled | default true }}
-            - name: PUBSUB_GROUP
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.labels['app.kubernetes.io/name']
-            {{- end }}
-            {{- if eq ((.Values.remoteControlPlane).api.authMode) "oauth" }}
-            - name: OAUTH_CLIENT_ID
-              valueFrom:
-                configMapKeyRef:
-                    name: job-service-config
-                    key: oauth_client_id
-            - name: OAUTH_CLIENT_SECRET
-              valueFrom:
-                secretKeyRef:
-                    name: job-service-keptn-secrets
-                    key: oauth_client_secret
-            - name: OAUTH_DISCOVERY
-              valueFrom:
-                configMapKeyRef:
-                    name: job-service-config
-                    key: oauth_discovery
-            - name: OAUTH_SCOPES
-              valueFrom:
-                configMapKeyRef:
-                    name: job-service-config
-                    key: oauth_scopes
-            {{- end }}
-            - name: API_PROXY_MAX_PAYLOAD_BYTES_KB
-              value: "128"
+                  key: oauth_client_secret
+          - name: OAUTH_DISCOVERY
+            valueFrom:
+              configMapKeyRef:
+                  name: job-service-config
+                  key: oauth_discovery
+          - name: OAUTH_SCOPES
+            valueFrom:
+              configMapKeyRef:
+                  name: job-service-config
+                  key: oauth_scopes
+          {{- end }}
+          - name: API_PROXY_MAX_PAYLOAD_BYTES_KB
+            value: "128"
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
       volumes:
         - name: job-executor-settings
           projected:

diff --git a/chart/values.yaml b/chart/values.yaml
@@ -6,6 +6,9 @@ image:
   # Overrides the image tag whose default is the chart appVersion.
   tag: ""
 
+subscription:
+  pubsubTopic: "sh.keptn.event.test.triggered,sh.keptn.event.deployment.triggered,sh.keptn.event.evaluation.triggered,sh.keptn.event.release.triggered,sh.keptn.event.approval.triggered,sh.keptn.event.action.triggered" # Sets the events the service subscribes to
+
 jobexecutorserviceinitcontainer:
   image:
     repository: docker.io/keptncontrib/job-executor-service-initcontainer # Container Image Name

diff --git a/cmd/job-executor-service-initcontainer/main.go b/cmd/job-executor-service-initcontainer/main.go
@@ -4,8 +4,9 @@ import (
 	"context"
 	oauthutils "github.com/keptn/go-utils/pkg/common/oauth2"
 	"golang.org/x/oauth2/clientcredentials"
+	"keptn-contrib/job-executor-service/pkg/config"
 	"keptn-contrib/job-executor-service/pkg/file"
-	"keptn-contrib/job-executor-service/pkg/keptn"
+	keptn_interface "keptn-contrib/job-executor-service/pkg/keptn"
 	"log"
 	"net/http"
 	"net/url"
@@ -14,6 +15,8 @@ import (
 
 	"github.com/kelseyhightower/envconfig"
 	api "github.com/keptn/go-utils/pkg/api/utils/v2"
+	//api "github.com/keptn/go-utils/pkg/api/utils/v2"
+	keptnv2 "github.com/keptn/go-utils/pkg/lib/v0_2_0"
 	"github.com/spf13/afero"
 )
 
@@ -24,7 +27,7 @@ const OAuthDiscoveryTimeout = 10 * time.Second
 type envConfig struct {
 	// Whether we are running locally (e.g., for testing) or on production
 	Env string `envconfig:"ENV" default:"local"`
-	// URL of the Keptn configuration service (this is where we can fetch files from the config repo)
+	// URL of the Keptn API Endpoint
 	KeptnAPIURL string `envconfig:"KEPTN_API_URL" required:"true"`
 	// The token of the keptn API
 	KeptnAPIToken string `envconfig:"KEPTN_API_TOKEN" required:"true"`
@@ -112,25 +115,20 @@ func main() {
 		log.Fatalf("unable to create keptn API: %s", err)
 	}
 
-	useLocalFileSystem := false
-
-	// configure keptn options
-	if env.Env == "local" {
-		log.Println("env=local: Running with local filesystem to fetch resources")
-		useLocalFileSystem = true
-	}
-
 	// re-create the event from job-executor-service
-	eventProps := keptn.EventProperties{
-		Project:     env.Project,
-		Stage:       env.Stage,
-		Service:     env.Service,
-		GitCommitID: env.GitCommitID,
+	eventProps := &keptnv2.EventData{
+		Project: env.Project,
+		Stage:   env.Stage,
+		Service: env.Service,
 	}
 
-	configService := keptn.NewConfigService(useLocalFileSystem, eventProps, keptnAPI.Resources())
+	resourceService := keptn_interface.NewV1ResourceHandler(*eventProps, keptnAPI.Resources())
+
+	jobConfigHandler := config.JobConfigReader{
+		Keptn: resourceService,
+	}
 
-	err = file.MountFiles(env.Action, env.Task, fs, configService)
+	err = file.MountFiles(env.Action, env.Task, env.GitCommitID, fs, jobConfigHandler)
 	if err != nil {
 		log.Printf("Error while copying files: %s", err.Error())
 		os.Exit(-1)