Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Ingress usage metrics #976

Merged
merged 1 commit into from
Jan 6, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pkg/context/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"k8s.io/ingress-gce/pkg/common/typed"
frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned"
informerfrontendconfig "k8s.io/ingress-gce/pkg/frontendconfig/client/informers/externalversions/frontendconfig/v1beta1"
"k8s.io/ingress-gce/pkg/metrics"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
"k8s.io/klog"
Expand Down Expand Up @@ -75,6 +76,8 @@ type ControllerContext struct {
DestinationRuleInformer cache.SharedIndexInformer
ConfigMapInformer cache.SharedIndexInformer

ControllerMetrics *metrics.ControllerMetrics

healthChecks map[string]func() error

lock sync.Mutex
Expand Down Expand Up @@ -114,6 +117,7 @@ func NewControllerContext(
Cloud: cloud,
ClusterNamer: namer,
KubeSystemUID: kubeSystemUID,
ControllerMetrics: metrics.NewControllerMetrics(),
ControllerContextConfig: config,
IngressInformer: informerv1beta1.NewIngressInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
ServiceInformer: informerv1.NewServiceInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
Expand Down Expand Up @@ -293,6 +297,8 @@ func (ctx *ControllerContext) Start(stopCh chan struct{}) {
if ctx.EnableASMConfigMap && ctx.ConfigMapInformer != nil {
go ctx.ConfigMapInformer.Run(stopCh)
}
// Export ingress usage metrics.
go ctx.ControllerMetrics.Run(stopCh)
skmatti marked this conversation as resolved.
Show resolved Hide resolved
}

// Ingresses returns the store of Ingresses.
Expand Down
12 changes: 12 additions & 0 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"k8s.io/ingress-gce/pkg/healthchecks"
"k8s.io/ingress-gce/pkg/instances"
"k8s.io/ingress-gce/pkg/loadbalancers"
"k8s.io/ingress-gce/pkg/metrics"
negtypes "k8s.io/ingress-gce/pkg/neg/types"
ingsync "k8s.io/ingress-gce/pkg/sync"
"k8s.io/ingress-gce/pkg/tls"
Expand Down Expand Up @@ -89,6 +90,9 @@ type LoadBalancerController struct {

// Ingress sync + GC implementation
ingSyncer ingsync.Syncer

// Ingress usage metrics.
metrics metrics.IngressMetricsCollector
}

// NewLoadBalancerController creates a controller for gce loadbalancers.
Expand Down Expand Up @@ -119,6 +123,7 @@ func NewLoadBalancerController(
backendSyncer: backends.NewBackendSyncer(backendPool, healthChecker, ctx.Cloud),
negLinker: backends.NewNEGLinker(backendPool, negtypes.NewAdapter(ctx.Cloud), ctx.Cloud),
igLinker: backends.NewInstanceGroupLinker(instancePool, backendPool),
metrics: ctx.ControllerMetrics,
}
lbc.ingSyncer = ingsync.NewIngressSyncer(&lbc)

Expand Down Expand Up @@ -531,6 +536,10 @@ func (lbc *LoadBalancerController) sync(key string) error {
if err != nil && ingExists {
lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "GC", fmt.Sprintf("Error during GC: %v", err))
}
// Delete the ingress state for metrics after GC is successful.
if err == nil && ingExists {
lbc.metrics.DeleteIngress(key)
}
return err
}

Expand All @@ -555,6 +564,9 @@ func (lbc *LoadBalancerController) sync(key string) error {
syncErr := lbc.ingSyncer.Sync(syncState)
if syncErr != nil {
lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "Sync", fmt.Sprintf("Error during sync: %v", syncErr.Error()))
} else {
// Insert/update the ingress state for metrics after successful sync.
lbc.metrics.SetIngress(key, metrics.NewIngressState(ing, urlMap.AllServicePorts()))
}

// Garbage collection will occur regardless of an error occurring. If an error occurred,
Expand Down
237 changes: 237 additions & 0 deletions pkg/metrics/features.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
"fmt"
"strconv"

"k8s.io/api/networking/v1beta1"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/klog"
)

type feature string

func (f feature) String() string {
return string(f)
}

const (
// WARNING: Please keep the following constants in sync with
// pkg/annotations/ingress.go
// allowHTTPKey tells the Ingress controller to allow/block HTTP access.
skmatti marked this conversation as resolved.
Show resolved Hide resolved
allowHTTPKey = "kubernetes.io/ingress.allow-http"
ingressClassKey = "kubernetes.io/ingress.class"
gceIngressClass = "gce"
gceMultiIngressClass = "gce-multi-cluster"
gceL7ILBIngressClass = "gce-internal"
// preSharedCertKey represents the specific pre-shared SSL
// certificate for the Ingress controller to use.
preSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
managedCertKey = "networking.gke.io/managed-certificates"
// staticIPKey is the annotation key used by controller to record GCP static ip.
staticIPKey = "ingress.kubernetes.io/static-ip"

ingress = feature("Ingress")
externalIngress = feature("ExternalIngress")
internalIngress = feature("InternalIngress")
httpEnabled = feature("HTTPEnabled")
hostBasedRouting = feature("HostBasedRouting")
pathBasedRouting = feature("PathBasedRouting")
tlsTermination = feature("TLSTermination")
secretBasedCertsForTLS = feature("SecretBasedCertsForTLS")
preSharedCertsForTLS = feature("PreSharedCertsForTLS")
managedCertsForTLS = feature("ManagedCertsForTLS")
staticGlobalIP = feature("StaticGlobalIP")

servicePort = feature("L7LBServicePort")
externalServicePort = feature("L7XLBServicePort")
internalServicePort = feature("L7ILBServicePort")
neg = feature("NEG")
cloudCDN = feature("CloudCDN")
cloudArmor = feature("CloudArmor")
cloudIAP = feature("CloudIAP")
backendTimeout = feature("BackendTimeout")
backendConnectionDraining = feature("BackendConnectionDraining")
clientIPAffinity = feature("ClientIPAffinity")
cookieAffinity = feature("CookieAffinity")
customRequestHeaders = feature("CustomRequestHeaders")
)

// featuresForIngress returns the list of features for given ingress.
func featuresForIngress(ing *v1beta1.Ingress) []feature {
features := []feature{ingress}

ingKey := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name)
klog.V(4).Infof("Listing features for Ingress %s", ingKey)
ingAnnotations := ing.Annotations

// Determine the type of ingress based on ingress class.
ingClass := ingAnnotations[ingressClassKey]
klog.V(6).Infof("Ingress class value for ingress %s: %s", ingKey, ingClass)
switch ingClass {
case "", gceIngressClass, gceMultiIngressClass:
features = append(features, externalIngress)
case gceL7ILBIngressClass:
features = append(features, internalIngress)
}

// Determine if http is enabled.
if val, ok := ingAnnotations[allowHTTPKey]; !ok {
klog.V(6).Infof("Annotation %s does not exist for ingress %s", allowHTTPKey, ingKey)
features = append(features, httpEnabled)
} else {
klog.V(6).Infof("User specified value for annotation %s on ingress %s: %s", allowHTTPKey, ingKey, val)
v, err := strconv.ParseBool(val)
if err != nil {
klog.Errorf("Failed to parse %s for annotation %s on ingress %s", val, allowHTTPKey, ingKey)
}
if err == nil && v {
features = append(features, httpEnabled)
}
}

// An ingress without a host or http-path is ignored.
hostBased, pathBased := false, false
if len(ing.Spec.Rules) == 0 {
klog.V(6).Infof("Neither host-based nor path-based routing rules are setup for ingress %s", ingKey)
}
for _, rule := range ing.Spec.Rules {
if rule.HTTP != nil && len(rule.HTTP.Paths) > 0 {
klog.V(6).Infof("User specified http paths for ingress %s: %v", ingKey, rule.HTTP.Paths)
pathBased = true
}
if rule.Host != "" {
klog.V(6).Infof("User specified host for ingress %s: %v", ingKey, rule.Host)
hostBased = true
}
if pathBased && hostBased {
break
}
}
if hostBased {
features = append(features, hostBasedRouting)
}
if pathBased {
features = append(features, pathBasedRouting)
}

// SSL certificate based features.
sslConfigured := false
if val, ok := ingAnnotations[preSharedCertKey]; ok {
klog.V(6).Infof("Specified pre-shared certs for ingress %s: %v", ingKey, val)
sslConfigured = true
features = append(features, preSharedCertsForTLS)
}
if val, ok := ingAnnotations[managedCertKey]; ok {
klog.V(6).Infof("Specified google managed certs for ingress %s: %v", ingKey, val)
sslConfigured = true
features = append(features, managedCertsForTLS)
}
if hasSecretBasedCerts(ing) {
sslConfigured = true
features = append(features, secretBasedCertsForTLS)
}
if sslConfigured {
klog.V(6).Infof("TLS termination is configured for ingress %s", ingKey)
features = append(features, tlsTermination)
}

// Both user specified and ingress controller managed global static ips are reported.
if val, ok := ingAnnotations[staticIPKey]; ok && val != "" {
klog.V(6).Infof("Specified static for ingress %s: %s", ingKey, val)
features = append(features, staticGlobalIP)
}
klog.V(4).Infof("Features for ingress %s/%s: %v", ing.Namespace, ing.Name, features)
return features
}

// hasSecretBasedCerts returns true if ingress spec contains a secret based cert.
func hasSecretBasedCerts(ing *v1beta1.Ingress) bool {
for _, tlsSecret := range ing.Spec.TLS {
if tlsSecret.SecretName == "" {
continue
}
klog.V(6).Infof("User specified secret for ingress %s/%s: %s", ing.Namespace, ing.Name, tlsSecret.SecretName)
return true
}
return false
}

// featuresForServicePort returns the list of features for given service port.
func featuresForServicePort(sp utils.ServicePort) []feature {
features := []feature{servicePort}
svcPortKey := newServicePortKey(sp).string()
klog.V(4).Infof("Listing features for service port %s", svcPortKey)
if sp.L7ILBEnabled {
klog.V(6).Infof("L7 ILB is enabled for service port %s", svcPortKey)
features = append(features, internalServicePort)
} else {
features = append(features, externalServicePort)
}
if sp.NEGEnabled {
klog.V(6).Infof("NEG is enabled for service port %s", svcPortKey)
features = append(features, neg)
}
if sp.BackendConfig == nil {
klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features)
return features
}

beConfig := fmt.Sprintf("%s/%s", sp.BackendConfig.Namespace, sp.BackendConfig.Name)
klog.V(6).Infof("Backend config specified for service port %s: %s", svcPortKey, beConfig)

if sp.BackendConfig.Spec.Cdn != nil && sp.BackendConfig.Spec.Cdn.Enabled {
klog.V(6).Infof("Cloud CDN is enabled for service port %s", svcPortKey)
features = append(features, cloudCDN)
}
if sp.BackendConfig.Spec.Iap != nil && sp.BackendConfig.Spec.Iap.Enabled {
klog.V(6).Infof("Cloud IAP is enabled for service port %s", svcPortKey)
features = append(features, cloudIAP)
}
// Possible list of Affinity types:
// NONE, CLIENT_IP, GENERATED_COOKIE, CLIENT_IP_PROTO, or CLIENT_IP_PORT_PROTO.
if sp.BackendConfig.Spec.SessionAffinity != nil {
affinityType := sp.BackendConfig.Spec.SessionAffinity.AffinityType
switch affinityType {
case "GENERATED_COOKIE":
features = append(features, cookieAffinity)
case "CLIENT_IP", "CLIENT_IP_PROTO", "CLIENT_IP_PORT_PROTO":
features = append(features, clientIPAffinity)
}
klog.V(6).Infof("Session affinity %s is configured for service port %s", affinityType, svcPortKey)
}
if sp.BackendConfig.Spec.SecurityPolicy != nil {
klog.V(6).Infof("Security policy %s is configured for service port %s", sp.BackendConfig.Spec.SecurityPolicy, svcPortKey)
features = append(features, cloudArmor)
}
if sp.BackendConfig.Spec.TimeoutSec != nil {
klog.V(6).Infof("Backend timeout(%v secs) is configured for service port %s", sp.BackendConfig.Spec.TimeoutSec, svcPortKey)
features = append(features, backendTimeout)
}
if sp.BackendConfig.Spec.ConnectionDraining != nil {
klog.V(6).Infof("Backend connection draining(%v secs) is configured for service port %s", sp.BackendConfig.Spec.ConnectionDraining.DrainingTimeoutSec, svcPortKey)
features = append(features, backendConnectionDraining)
}
if sp.BackendConfig.Spec.CustomRequestHeaders != nil {
klog.V(6).Infof("Custom request headers configured for service port %s: %v", svcPortKey, sp.BackendConfig.Spec.CustomRequestHeaders.Headers)
features = append(features, customRequestHeaders)
}
klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features)
return features
}
Loading