From e0268c40281632e3758c6ad3e0527ce50e05c730 Mon Sep 17 00:00:00 2001 From: Andrew Seigner Date: Wed, 27 Feb 2019 13:58:01 -0800 Subject: [PATCH] Skip outbound port 443 in control-plane linkerd/linkerd2#2349 introduced a `SelfSubjectAccessReview` check at startup, to determine whether each control-plane component should establish Kubernetes watches cluster-wide or namespace-wide. If this check occurs before the linkerd-proxy sidecar is ready, it fails, and the control-plane component restarts. This change configures each control-plane pod to skip outbound port 443 when injecting the proxy, allowing the control-plane to connect to Kubernetes regardless of the `linkerd-proxy` state. A longer-term fix should involve a more robust control-plane startup, that is resilient to failed Kubernetes API requests. An even longer-term fix could involve injecting `linkerd-proxy` as a Kubernetes "sidecar" container, when that becomes available. Workaround for #2407 Signed-off-by: Andrew Seigner --- cli/cmd/install.go | 6 ++++++ cli/cmd/testdata/install_default.golden | 8 ++++++++ cli/cmd/testdata/install_ha_output.golden | 8 ++++++++ .../testdata/install_ha_with_overrides_output.golden | 8 ++++++++ cli/cmd/testdata/install_output.golden | 12 ++++++++++++ .../testdata/install_single_namespace_output.golden | 10 ++++++++++ 6 files changed, 52 insertions(+) diff --git a/cli/cmd/install.go b/cli/cmd/install.go index 8fb3f43574f96..42d633b6f9fb2 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -157,6 +157,7 @@ func validateAndBuildConfig(options *installOptions) (*installConfig, error) { return nil, err } + // TODO: these seem to not be used? ignoreInboundPorts := []string{ fmt.Sprintf("%d", options.proxyControlPort), fmt.Sprintf("%d", options.proxyMetricsPort), @@ -321,6 +322,11 @@ func render(config installConfig, w io.Writer, options *installOptions) error { // Special case for linkerd-proxy running in the Prometheus pod. injectOptions.proxyOutboundCapacity[config.PrometheusImage] = prometheusProxyOutboundCapacity + // Skip outbound port 443 to enable Kubernetes API access without the proxy. + // Once Kubernetes supports sidecar containers, this may be removed, as that + // will guarantee the proxy is running prior to control-plane startup. + injectOptions.ignoreOutboundPorts = []uint{443} + return InjectYAML(&buf, w, ioutil.Discard, injectOptions) } diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden index cd66c428ce0d5..9f7dbe5d36702 100644 --- a/cli/cmd/testdata/install_default.golden +++ b/cli/cmd/testdata/install_default.golden @@ -240,6 +240,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -518,6 +520,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -699,6 +703,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -946,6 +952,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init diff --git a/cli/cmd/testdata/install_ha_output.golden b/cli/cmd/testdata/install_ha_output.golden index 5a0e51506686b..27b0cc9386883 100644 --- a/cli/cmd/testdata/install_ha_output.golden +++ b/cli/cmd/testdata/install_ha_output.golden @@ -252,6 +252,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -536,6 +538,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -723,6 +727,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -976,6 +982,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init diff --git a/cli/cmd/testdata/install_ha_with_overrides_output.golden b/cli/cmd/testdata/install_ha_with_overrides_output.golden index 1ab57e1867374..9c79ebf349ad1 100644 --- a/cli/cmd/testdata/install_ha_with_overrides_output.golden +++ b/cli/cmd/testdata/install_ha_with_overrides_output.golden @@ -252,6 +252,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -536,6 +538,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -723,6 +727,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -976,6 +982,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden index 4d75a03f8d256..62ac0a057220a 100644 --- a/cli/cmd/testdata/install_output.golden +++ b/cli/cmd/testdata/install_output.golden @@ -243,6 +243,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -522,6 +524,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -704,6 +708,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -952,6 +958,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -1189,6 +1197,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -1323,6 +1333,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init diff --git a/cli/cmd/testdata/install_single_namespace_output.golden b/cli/cmd/testdata/install_single_namespace_output.golden index d18e7b34ee218..8ea766e27e372 100644 --- a/cli/cmd/testdata/install_single_namespace_output.golden +++ b/cli/cmd/testdata/install_single_namespace_output.golden @@ -239,6 +239,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -412,6 +414,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -596,6 +600,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -846,6 +852,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init @@ -1085,6 +1093,8 @@ spec: - "2102" - --inbound-ports-to-ignore - 4190,4191 + - --outbound-ports-to-ignore + - "443" image: gcr.io/linkerd-io/proxy-init:dev-undefined imagePullPolicy: IfNotPresent name: linkerd-init