Skip to content

Commit

Permalink
emit warning on no liveness probe defined for pods
Browse files Browse the repository at this point in the history
This patch iterates through pod specs, listing specs with no liveness
probes set. Any replication controllers whose deployment is fulfilled by
a DeploymentConfig are ignored.

**Example**
```
$ oc create -f pkg/api/graph/test/simple-deployment
deploymentconfig "simple-deployment" created

$ oc get all
NAME                   REVISION   DESIRED   CURRENT   TRIGGERED BY
dc/simple-deployment   1          1         0         config

NAME                     DESIRED   CURRENT   READY     AGE
rc/simple-deployment-1   0         0         0         1m

NAME                            READY     STATUS              RESTARTS
AGE
po/simple-deployment-1-deploy   0/1       ContainerCreating   0
1m

$ oc status -v
In project test on server https://10.111.123.56:8443

dc/simple-deployment deploys docker.io/openshift/deployment-example:v1
  deployment #1 pending 26 seconds ago

Warnings:
  * pod/simple-deployment-1-deploy has no liveness probe to verify pods are still running.
    try: oc set probe pod/simple-deployment-1-deploy --liveness ...
  * dc/simple-deployment has no liveness probe to verify pods are still running.
    try: oc set probe dc/simple-deployment --liveness ...

View details with 'oc describe <resource>/<name>' or list everything with 'oc get all'.
```
  • Loading branch information
juanvallejo committed Oct 14, 2016
1 parent 8f6030a commit 977778b
Show file tree
Hide file tree
Showing 4 changed files with 129 additions and 2 deletions.
87 changes: 85 additions & 2 deletions pkg/api/kubegraph/analysis/podspec.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,16 @@ import (
osgraph "github.com/openshift/origin/pkg/api/graph"
kubeedges "github.com/openshift/origin/pkg/api/kubegraph"
kubegraph "github.com/openshift/origin/pkg/api/kubegraph/nodes"
kubenodes "github.com/openshift/origin/pkg/api/kubegraph/nodes"
deploygraph "github.com/openshift/origin/pkg/deploy/graph"
deploynodes "github.com/openshift/origin/pkg/deploy/graph/nodes"
"k8s.io/kubernetes/pkg/util/sets"
)

const (
UnmountableSecretWarning = "UnmountableSecret"
MissingSecretWarning = "MissingSecret"
UnmountableSecretWarning = "UnmountableSecret"
MissingSecretWarning = "MissingSecret"
MissingLivenessProbeWarning = "MissingLivenessProbe"
)

// FindUnmountableSecrets inspects all PodSpecs for any Secret reference that isn't listed as mountable by the referenced ServiceAccount
Expand Down Expand Up @@ -75,6 +80,65 @@ func FindMissingSecrets(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
return markers
}

// FindMissingLivenessProbes inspects all PodSpecs for missing liveness probes and generates a list of non-duplicate markers
func FindMissingLivenessProbes(g osgraph.Graph, f osgraph.Namer, setProbeCommand string) []osgraph.Marker {
markers := []osgraph.Marker{}
appendedNodes := sets.NewString()
ignoredNodes := findDeploymentEdgeKinds(g)

for _, uncastPodSpecNode := range g.NodesByKind(kubegraph.PodSpecNodeKind) {
podSpecNode := uncastPodSpecNode.(*kubegraph.PodSpecNode)
podsWithoutLivenessProbes := CheckForLivenessProbes(g, podSpecNode)

topLevelNode := osgraph.GetTopLevelContainerNode(g, podSpecNode)
topLevelString := f.ResourceName(topLevelNode)

// prevent duplicate markers
if appendedNodes.Has(topLevelString) {
continue
}

// ignore replication controllers owned by DeploymentConfigs
if isNodeInList(topLevelNode, ignoredNodes) {
continue
}

appendedNodes.Insert(topLevelString)
for _, node := range podsWithoutLivenessProbes {
markers = append(markers, osgraph.Marker{
Node: podSpecNode,
RelatedNodes: []graph.Node{node},

Severity: osgraph.WarningSeverity,
Key: MissingLivenessProbeWarning,
Message: fmt.Sprintf("%s has no liveness probe to verify pods are still running.",
topLevelString),
Suggestion: osgraph.Suggestion(fmt.Sprintf("%s %s --liveness ...", setProbeCommand, topLevelString)),
})
}
}

return markers
}

func CheckForLivenessProbes(g osgraph.Graph, podSpecNode *kubegraph.PodSpecNode) []*kubegraph.PodSpecNode {
noLivenessProbes := []*kubegraph.PodSpecNode{}

hasLivenessProbe := false
for _, container := range podSpecNode.PodSpec.Containers {
if container.LivenessProbe != nil {
hasLivenessProbe = true
break
}
}

if !hasLivenessProbe {
noLivenessProbes = append(noLivenessProbes, podSpecNode)
}

return noLivenessProbes
}

// CheckForUnmountableSecrets checks to be sure that all the referenced secrets are mountable (by service account)
func CheckForUnmountableSecrets(g osgraph.Graph, podSpecNode *kubegraph.PodSpecNode) []*kubegraph.SecretNode {
saNodes := g.SuccessorNodesByNodeAndEdgeKind(podSpecNode, kubegraph.ServiceAccountNodeKind, kubeedges.ReferencedServiceAccountEdgeKind)
Expand Down Expand Up @@ -122,3 +186,22 @@ func CheckMissingMountedSecrets(g osgraph.Graph, podSpecNode *kubegraph.PodSpecN

return missingSecrets
}

// findDeploymentEdgeKinds returns all replication controller nodes
// whose deployment is being fulfilled by a DeploymentConfig
func findDeploymentEdgeKinds(g osgraph.Graph) []graph.Node {
nodeFilter := osgraph.NodesOfKind(deploynodes.DeploymentConfigNodeKind)
edgeFilter := osgraph.EdgesOfKind(deploygraph.DeploymentEdgeKind)

subGraph := g.Subgraph(nodeFilter, edgeFilter)
return subGraph.NodesByKind(kubenodes.ReplicationControllerNodeKind)
}

func isNodeInList(node graph.Node, nodeList []graph.Node) bool {
for _, n := range nodeList {
if n == node {
return true
}
}
return false
}
20 changes: 20 additions & 0 deletions pkg/api/kubegraph/analysis/podspec_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,23 @@ func TestUnmountableSecrets(t *testing.T) {
t.Errorf("expected %v, got %v", expectedSecret2, markers)
}
}

func TestMissingLivenessProbes(t *testing.T) {
g, _, err := osgraphtest.BuildGraph("../../../api/graph/test/simple-deployment.yaml")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}

kubeedges.AddAllExposedPodEdges(g)

markers := FindMissingLivenessProbes(g, osgraph.DefaultNamer, "oc set probe")
if e, a := 1, len(markers); e != a {
t.Fatalf("expected %v, got %v", e, a)
}

actualDC := osgraph.GetTopLevelContainerNode(g, markers[0].Node)
expectedDC := g.Find(osgraph.UniqueName("DeploymentConfig|/simple-deployment"))
if e, a := expectedDC.ID(), actualDC.ID(); e != a {
t.Errorf("expected %v, got %v", e, a)
}
}
3 changes: 3 additions & 0 deletions pkg/cmd/cli/describe/projectstatus.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,9 @@ func getMarkerScanners(logsCommandName, securityPolicyCommandFormat, setProbeCom
func(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
return deployanalysis.FindDeploymentConfigReadinessWarnings(g, f, setProbeCommandName)
},
func(g osgraph.Graph, f osgraph.Namer) []osgraph.Marker {
return kubeanalysis.FindMissingLivenessProbes(g, f, setProbeCommandName)
},
routeanalysis.FindPortMappingIssues,
routeanalysis.FindMissingTLSTerminationType,
routeanalysis.FindPathBasedPassthroughRoutes,
Expand Down
21 changes: 21 additions & 0 deletions test/cmd/set-liveness-probe.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
#!/bin/bash
source "$(dirname "${BASH_SOURCE}")/../../hack/lib/init.sh"
trap os::test::junit::reconcile_output EXIT

# Cleanup cluster resources created by this test
(
set +e
oc delete all,templates --all
exit 0
) &>/dev/null


os::test::junit::declare_suite_start "cmd/set-probe-liveness"
# This test setting a liveness probe, without warning about replication controllers whose deployment depends on deployment configs
os::cmd::expect_success_and_text 'oc create -f pkg/api/graph/test/simple-deployment.yaml' 'deploymentconfig "simple-deployment" created'
os::cmd::expect_success_and_text 'oc status -v' 'dc/simple-deployment has no liveness probe'
os::cmd::expect_success_and_not_text 'oc status -v' 'rc/simple-deployment-1 has no liveness probe'
os::cmd::expect_success_and_text 'oc set probe dc/simple-deployment --liveness --get-url=http://google.com:80' 'deploymentconfig "simple-deployment" updated'
os::cmd::expect_success_and_not_text 'oc status -v' 'dc/simple-deployment has no liveness probe'
echo "set-probe-liveness: ok"
os::test::junit::declare_suite_end

0 comments on commit 977778b

Please sign in to comment.