From aa9bc223557be3652ca26b225be99b5725a11de6 Mon Sep 17 00:00:00 2001 From: Andrew Seigner Date: Wed, 27 Feb 2019 15:06:25 -0800 Subject: [PATCH] Introduce logging and restart integration tests The integration tests deploy complete Linkerd environments into Kubernetes, but do not check if the components are logging errors or restarting. Introduce integration tests to validation that all expected control-plane containers (including `linkerd-proxy` and `linkerd-init`) are found, logging no errors, and not restarting. Fixes #2348 Signed-off-by: Andrew Seigner --- test/install_test.go | 121 +++++++++++++++++++++++++++++++--- testutil/kubernetes_helper.go | 7 ++ 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/test/install_test.go b/test/install_test.go index d07ab079b91d4..61bf91f08f9f2 100644 --- a/test/install_test.go +++ b/test/install_test.go @@ -3,6 +3,7 @@ package test import ( "fmt" "os" + "regexp" "strconv" "strings" "testing" @@ -11,6 +12,16 @@ import ( "github.com/linkerd/linkerd2/testutil" ) +type deploySpec struct { + replicas int + containers []string +} + +const ( + proxyContainer = "linkerd-proxy" + initContainer = "linkerd-init" +) + ////////////////////// /// TEST SETUP /// ////////////////////// @@ -31,11 +42,21 @@ var ( "linkerd-web", } - linkerdDeployReplicas = map[string]int{ - "linkerd-controller": 1, - "linkerd-grafana": 1, - "linkerd-prometheus": 1, - "linkerd-web": 1, + linkerdDeployReplicas = map[string]deploySpec{ + "linkerd-controller": {1, []string{"destination", "public-api", "tap"}}, + "linkerd-grafana": {1, []string{}}, + "linkerd-prometheus": {1, []string{}}, + "linkerd-web": {1, []string{"web"}}, + } + + // linkerd-proxy logs some errors when TLS is enabled, remove these once + // they're addressed. + knownProxyErrors = []string{ + `linkerd2_proxy::control::serve_http error serving metrics: Error { kind: Shutdown, cause: Os { code: 107, kind: NotConnected, message: "Transport endpoint is not connected" } }`, + `linkerd-proxy ERR! admin={bg=tls-config} linkerd2_proxy::transport::tls::config error loading /var/linkerd-io/identity/certificate.crt: No such file or directory (os error 2)`, + `linkerd-proxy ERR! admin={bg=tls-config} linkerd2_proxy::transport::tls::config error loading /var/linkerd-io/trust-anchors/trust-anchors.pem: No such file or directory (os error 2)`, + `linkerd-proxy WARN admin={bg=tls-config} linkerd2_proxy::transport::tls::config error reloading TLS config: Io("/var/linkerd-io/identity/certificate.crt", Some(2)), falling back`, + `linkerd-proxy WARN admin={bg=tls-config} linkerd2_proxy::transport::tls::config error reloading TLS config: Io("/var/linkerd-io/trust-anchors/trust-anchors.pem", Some(2)), falling back`, } ) @@ -83,7 +104,7 @@ func TestInstall(t *testing.T) { } if TestHelper.TLS() { cmd = append(cmd, []string{"--tls", "optional"}...) - linkerdDeployReplicas["linkerd-ca"] = 1 + linkerdDeployReplicas["linkerd-ca"] = deploySpec{1, []string{"ca"}} } if TestHelper.SingleNamespace() { cmd = append(cmd, "--single-namespace") @@ -113,11 +134,11 @@ func TestInstall(t *testing.T) { } // Tests Pods and Deployments - for deploy, replicas := range linkerdDeployReplicas { - if err := TestHelper.CheckPods(TestHelper.GetLinkerdNamespace(), deploy, replicas); err != nil { + for deploy, spec := range linkerdDeployReplicas { + if err := TestHelper.CheckPods(TestHelper.GetLinkerdNamespace(), deploy, spec.replicas); err != nil { t.Fatal(fmt.Errorf("Error validating pods for deploy [%s]:\n%s", deploy, err)) } - if err := TestHelper.CheckDeployment(TestHelper.GetLinkerdNamespace(), deploy, replicas); err != nil { + if err := TestHelper.CheckDeployment(TestHelper.GetLinkerdNamespace(), deploy, spec.replicas); err != nil { t.Fatal(fmt.Errorf("Error validating deploy [%s]:\n%s", deploy, err)) } } @@ -261,3 +282,85 @@ func TestCheckProxy(t *testing.T) { t.Fatal(err.Error()) } } + +func TestLogs(t *testing.T) { + controllerRegex := regexp.MustCompile("level=(panic|fatal|error|warn)") + proxyRegex := regexp.MustCompile(fmt.Sprintf("%s (ERR|WARN)", proxyContainer)) + + for deploy, spec := range linkerdDeployReplicas { + deploy := strings.TrimPrefix(deploy, "linkerd-") + containers := append(spec.containers, proxyContainer) + + for _, container := range containers { + regex := controllerRegex + if container == proxyContainer { + regex = proxyRegex + } + + outputStream, err := TestHelper.LinkerdRunStream( + "logs", "--no-color", + "--control-plane-component", deploy, + "--container", container, + ) + if err != nil { + t.Fatalf("Error running command:\n%s", err) + } + defer outputStream.Stop() + outputLines, _ := outputStream.ReadUntil(100, 10*time.Second) + if len(outputLines) == 0 { + t.Fatalf("No logs found for %s/%s", deploy, container) + } + + for _, line := range outputLines { + if regex.MatchString(line) { + + // check for known errors + known := false + if TestHelper.TLS() && container == proxyContainer { + for _, er := range knownProxyErrors { + if strings.HasSuffix(line, er) { + known = true + break + } + } + } + + if !known { + t.Fatalf("Found error in %s/%s log: %s", deploy, container, line) + } + } + } + } + } +} + +func TestRestarts(t *testing.T) { + for deploy, spec := range linkerdDeployReplicas { + deploy := strings.TrimPrefix(deploy, "linkerd-") + containers := append(spec.containers, proxyContainer, initContainer) + + for _, container := range containers { + selector := fmt.Sprintf("linkerd.io/control-plane-component=%s", deploy) + containerStatus := "containerStatuses" + if container == initContainer { + containerStatus = "initContainerStatuses" + } + output := fmt.Sprintf("jsonpath='{.items[*].status.%s[?(@.name==\"%s\")].restartCount}'", containerStatus, container) + + out, err := TestHelper.Kubectl( + "-n", TestHelper.GetLinkerdNamespace(), + "get", "pods", + "--selector", selector, + "-o", output, + ) + if err != nil { + t.Fatalf("kubectl command failed\n%s", out) + } + if out == "''" { + t.Fatalf("Could not find restartCount for %s/%s", deploy, container) + } else if out != "'0'" { + t.Fatalf("Found %s restarts of %s/%s", out, deploy, container) + } + } + } +} diff --git a/testutil/kubernetes_helper.go b/testutil/kubernetes_helper.go index 4689d40ec98d3..792626dad270b 100644 --- a/testutil/kubernetes_helper.go +++ b/testutil/kubernetes_helper.go @@ -87,6 +87,13 @@ func (h *KubernetesHelper) KubectlApply(stdin string, namespace string) (string, return string(out), err } +// Kubectl executes an arbitrary Kubectl command +func (h *KubernetesHelper) Kubectl(arg ...string) (string, error) { + cmd := exec.Command("kubectl", arg...) + out, err := cmd.CombinedOutput() + return string(out), err +} + // getDeployments gets all deployments with a count of their ready replicas in // the specified namespace. func (h *KubernetesHelper) getDeployments(namespace string) (map[string]int, error) {