Add controller admin servers and readiness probes (#1168)

* Add controller admin servers and readiness probes * Tweak readiness probes to be more sane * Refactor based on review feedback Signed-off-by: Kevin Lingerfelt <kl@buoyant.io>
linkerd · Jun 21, 2018 · 682b027 · 682b027
1 parent c52385d
commit 682b027
Show file tree

Hide file tree

Showing 18 changed files with 261 additions and 73 deletions.
diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden
@@ -148,12 +148,22 @@ spec:
         - -logtostderr=true
         image: gcr.io/runconduit/controller:undefined
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9995
+          initialDelaySeconds: 10
         name: public-api
         ports:
         - containerPort: 8085
           name: http
         - containerPort: 9995
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9995
         resources: {}
       - args:
         - destination
@@ -162,12 +172,22 @@ spec:
         - -logtostderr=true
         image: gcr.io/runconduit/controller:undefined
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9999
+          initialDelaySeconds: 10
         name: destination
         ports:
         - containerPort: 8089
           name: grpc
         - containerPort: 9999
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9999
         resources: {}
       - args:
         - proxy-api
@@ -176,25 +196,45 @@ spec:
         - -logtostderr=true
         image: gcr.io/runconduit/controller:undefined
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9996
+          initialDelaySeconds: 10
         name: proxy-api
         ports:
         - containerPort: 8086
           name: grpc
         - containerPort: 9996
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9996
         resources: {}
       - args:
         - tap
         - -log-level=info
         - -logtostderr=true
         image: gcr.io/runconduit/controller:undefined
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9998
+          initialDelaySeconds: 10
         name: tap
         ports:
         - containerPort: 8088
           name: grpc
         - containerPort: 9998
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9998
         resources: {}
       - env:
         - name: CONDUIT_PROXY_LOG
@@ -306,12 +346,22 @@ spec:
         - -log-level=info
         image: gcr.io/runconduit/web:undefined
         imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9994
+          initialDelaySeconds: 10
         name: web
         ports:
         - containerPort: 8084
           name: http
         - containerPort: 9994
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9994
         resources: {}
       - env:
         - name: CONDUIT_PROXY_LOG
@@ -666,7 +716,7 @@ spec:
           httpGet:
             path: /api/health
             port: 3000
-          initialDelaySeconds: 60
+          initialDelaySeconds: 30
           periodSeconds: 10
           timeoutSeconds: 30
         resources: {}

diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden
@@ -149,12 +149,22 @@ spec:
         - -logtostderr=true
         image: ControllerImage
         imagePullPolicy: ImagePullPolicy
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9995
+          initialDelaySeconds: 10
         name: public-api
         ports:
         - containerPort: 8085
           name: http
         - containerPort: 9995
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9995
         resources: {}
       - args:
         - destination
@@ -163,12 +173,22 @@ spec:
         - -logtostderr=true
         image: ControllerImage
         imagePullPolicy: ImagePullPolicy
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9999
+          initialDelaySeconds: 10
         name: destination
         ports:
         - containerPort: 8089
           name: grpc
         - containerPort: 9999
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9999
         resources: {}
       - args:
         - proxy-api
@@ -177,25 +197,45 @@ spec:
         - -logtostderr=true
         image: ControllerImage
         imagePullPolicy: ImagePullPolicy
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9996
+          initialDelaySeconds: 10
         name: proxy-api
         ports:
         - containerPort: 123
           name: grpc
         - containerPort: 9996
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9996
         resources: {}
       - args:
         - tap
         - -log-level=ControllerLogLevel
         - -logtostderr=true
         image: ControllerImage
         imagePullPolicy: ImagePullPolicy
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9998
+          initialDelaySeconds: 10
         name: tap
         ports:
         - containerPort: 8088
           name: grpc
         - containerPort: 9998
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9998
         resources: {}
       - env:
         - name: CONDUIT_PROXY_LOG
@@ -308,12 +348,22 @@ spec:
         - -log-level=ControllerLogLevel
         image: WebImage
         imagePullPolicy: ImagePullPolicy
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9994
+          initialDelaySeconds: 10
         name: web
         ports:
         - containerPort: 8084
           name: http
         - containerPort: 9994
           name: admin-http
+        readinessProbe:
+          failureThreshold: 7
+          httpGet:
+            path: /ready
+            port: 9994
         resources: {}
       - env:
         - name: CONDUIT_PROXY_LOG
@@ -670,7 +720,7 @@ spec:
           httpGet:
             path: /api/health
             port: 3000
-          initialDelaySeconds: 60
+          initialDelaySeconds: 30
           periodSeconds: 10
           timeoutSeconds: 30
         resources: {}

diff --git a/cli/install/template.go b/cli/install/template.go
@@ -152,6 +152,16 @@ spec:
         - "-controller-namespace={{.Namespace}}"
         - "-log-level={{.ControllerLogLevel}}"
         - "-logtostderr=true"
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9995
+          initialDelaySeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 9995
+          failureThreshold: 7
       - name: destination
         ports:
         - name: grpc
@@ -165,6 +175,16 @@ spec:
         - "-enable-tls={{.EnableTLS}}"
         - "-log-level={{.ControllerLogLevel}}"
         - "-logtostderr=true"
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9999
+          initialDelaySeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 9999
+          failureThreshold: 7
       - name: proxy-api
         ports:
         - name: grpc
@@ -178,6 +198,16 @@ spec:
         - "-addr=:{{.ProxyAPIPort}}"
         - "-log-level={{.ControllerLogLevel}}"
         - "-logtostderr=true"
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9996
+          initialDelaySeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 9996
+          failureThreshold: 7
       - name: tap
         ports:
         - name: grpc
@@ -190,6 +220,16 @@ spec:
         - "tap"
         - "-log-level={{.ControllerLogLevel}}"
         - "-logtostderr=true"
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9998
+          initialDelaySeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 9998
+          failureThreshold: 7
 
 ### Web ###
 ---
@@ -249,6 +289,16 @@ spec:
         - "-uuid={{.UUID}}"
         - "-controller-namespace={{.Namespace}}"
         - "-log-level={{.ControllerLogLevel}}"
+        livenessProbe:
+          httpGet:
+            path: /ping
+            port: 9994
+          initialDelaySeconds: 10
+        readinessProbe:
+          httpGet:
+            path: /ready
+            port: 9994
+          failureThreshold: 7
 
 ### Prometheus ###
 ---
@@ -502,7 +552,7 @@ spec:
           httpGet:
             path: /api/health
             port: 3000
-          initialDelaySeconds: 60
+          initialDelaySeconds: 30
           timeoutSeconds: 30
           failureThreshold: 10
           periodSeconds: 10

diff --git a/controller/api/public/grpc_server_test.go b/controller/api/public/grpc_server_test.go
@@ -161,10 +161,7 @@ spec:
 				[]string{},
 			)
 
-			err = k8sAPI.Sync()
-			if err != nil {
-				t.Fatalf("k8sAPI.Sync() returned an error: %s", err)
-			}
+			k8sAPI.Sync(nil)
 
 			rsp, err := fakeGrpcServer.ListPods(context.TODO(), &pb.Empty{})
 			if err != exp.err {

diff --git a/controller/api/public/stat_summary_test.go b/controller/api/public/stat_summary_test.go
@@ -69,10 +69,7 @@ func testStatSummary(t *testing.T, expectations []statSumExpected) {
 			[]string{},
 		)
 
-		err = k8sAPI.Sync()
-		if err != nil {
-			t.Fatalf("k8sAPI.Sync() returned an error: %s", err)
-		}
+		k8sAPI.Sync(nil)
 
 		rsp, err := fakeGrpcServer.StatSummary(context.TODO(), &exp.req)
 		if err != exp.err {