Skip to content

Commit

Permalink
Add Ingress usage metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
skmatti committed Jan 2, 2020
1 parent 1d46157 commit 3fd78ad
Show file tree
Hide file tree
Showing 15 changed files with 1,263 additions and 9 deletions.
6 changes: 5 additions & 1 deletion cmd/glbc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (

ingctx "k8s.io/ingress-gce/pkg/context"
"k8s.io/ingress-gce/pkg/controller"
"k8s.io/ingress-gce/pkg/metrics"
"k8s.io/ingress-gce/pkg/neg"
negtypes "k8s.io/ingress-gce/pkg/neg/types"

Expand Down Expand Up @@ -131,6 +132,9 @@ func main() {
}
kubeSystemUID := kubeSystemNS.GetUID()

// Initialize ingress usage metrics.
ingresMetrics := metrics.NewIngressMetrics()

cloud := app.NewGCEClient()
defaultBackendServicePort := app.DefaultBackendServicePort(kubeClient)
ctxConfig := ingctx.ControllerContextConfig{
Expand All @@ -144,7 +148,7 @@ func main() {
ASMConfigMapNamespace: flags.F.ASMConfigMapBasedConfigNamespace,
ASMConfigMapName: flags.F.ASMConfigMapBasedConfigCMName,
}
ctx := ingctx.NewControllerContext(kubeConfig, kubeClient, backendConfigClient, frontendConfigClient, cloud, namer, kubeSystemUID, ctxConfig)
ctx := ingctx.NewControllerContext(kubeConfig, kubeClient, backendConfigClient, frontendConfigClient, cloud, namer, kubeSystemUID, ingresMetrics, ctxConfig)
go app.RunHTTPServer(ctx.HealthCheck)

if !flags.F.LeaderElection.LeaderElect {
Expand Down
7 changes: 7 additions & 0 deletions pkg/context/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"k8s.io/ingress-gce/pkg/common/typed"
frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned"
informerfrontendconfig "k8s.io/ingress-gce/pkg/frontendconfig/client/informers/externalversions/frontendconfig/v1beta1"
"k8s.io/ingress-gce/pkg/metrics"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
"k8s.io/klog"
Expand Down Expand Up @@ -75,6 +76,8 @@ type ControllerContext struct {
DestinationRuleInformer cache.SharedIndexInformer
ConfigMapInformer cache.SharedIndexInformer

IngressMetrics *metrics.IngressMetrics

healthChecks map[string]func() error

lock sync.Mutex
Expand Down Expand Up @@ -106,6 +109,7 @@ func NewControllerContext(
cloud *gce.Cloud,
namer *namer.Namer,
kubeSystemUID types.UID,
ingressMetrics *metrics.IngressMetrics,
config ControllerContextConfig) *ControllerContext {

context := &ControllerContext{
Expand All @@ -114,6 +118,7 @@ func NewControllerContext(
Cloud: cloud,
ClusterNamer: namer,
KubeSystemUID: kubeSystemUID,
IngressMetrics: ingressMetrics,
ControllerContextConfig: config,
IngressInformer: informerv1beta1.NewIngressInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
ServiceInformer: informerv1.NewServiceInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
Expand Down Expand Up @@ -293,6 +298,8 @@ func (ctx *ControllerContext) Start(stopCh chan struct{}) {
if ctx.EnableASMConfigMap && ctx.ConfigMapInformer != nil {
go ctx.ConfigMapInformer.Run(stopCh)
}
// Export ingress usage metrics.
go ctx.IngressMetrics.Run(stopCh)
}

// Ingresses returns the store of Ingresses.
Expand Down
8 changes: 8 additions & 0 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"k8s.io/ingress-gce/pkg/healthchecks"
"k8s.io/ingress-gce/pkg/instances"
"k8s.io/ingress-gce/pkg/loadbalancers"
"k8s.io/ingress-gce/pkg/metrics"
negtypes "k8s.io/ingress-gce/pkg/neg/types"
ingsync "k8s.io/ingress-gce/pkg/sync"
"k8s.io/ingress-gce/pkg/tls"
Expand Down Expand Up @@ -531,6 +532,10 @@ func (lbc *LoadBalancerController) sync(key string) error {
if err != nil && ingExists {
lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "GC", fmt.Sprintf("Error during GC: %v", err))
}
// Delete the ingress state for metrics after GC is successful.
if err == nil && ingExists {
lbc.ctx.IngressMetrics.Delete(key)
}
return err
}

Expand All @@ -555,6 +560,9 @@ func (lbc *LoadBalancerController) sync(key string) error {
syncErr := lbc.ingSyncer.Sync(syncState)
if syncErr != nil {
lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "Sync", fmt.Sprintf("Error during sync: %v", syncErr.Error()))
} else {
// Insert/update the ingress state for metrics after successful sync.
lbc.ctx.IngressMetrics.Set(key, metrics.NewIngressState(ing, urlMap.AllServicePorts()))
}

// Garbage collection will occur regardless of an error occurring. If an error occurred,
Expand Down
3 changes: 2 additions & 1 deletion pkg/controller/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"k8s.io/ingress-gce/pkg/flags"
"k8s.io/ingress-gce/pkg/instances"
"k8s.io/ingress-gce/pkg/loadbalancers"
"k8s.io/ingress-gce/pkg/metrics"
"k8s.io/ingress-gce/pkg/test"
"k8s.io/ingress-gce/pkg/tls"
"k8s.io/ingress-gce/pkg/utils"
Expand Down Expand Up @@ -72,7 +73,7 @@ func newLoadBalancerController() *LoadBalancerController {
HealthCheckPath: "/",
DefaultBackendHealthCheckPath: "/healthz",
}
ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, namer, "" /*kubeSystemUID*/, ctxConfig)
ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, namer, "" /*kubeSystemUID*/, metrics.NewIngressMetrics(), ctxConfig)
lbc := NewLoadBalancerController(ctx, stopCh)
// TODO(rramkumar): Fix this so we don't have to override with our fake
lbc.instancePool = instances.NewNodePool(instances.NewFakeInstanceGroups(sets.NewString(), namer), namer)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/translator/translator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func fakeTranslator() *Translator {
HealthCheckPath: "/",
DefaultBackendHealthCheckPath: "/healthz",
}
ctx := context.NewControllerContext(nil, client, backendConfigClient, nil, nil, defaultNamer, "" /*kubeSystemUID*/, ctxConfig)
ctx := context.NewControllerContext(nil, client, backendConfigClient, nil, nil, defaultNamer, "" /*kubeSystemUID*/, nil /*IngressMetrics*/, ctxConfig)
gce := &Translator{
ctx: ctx,
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/firewalls/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func newFirewallController() *FirewallController {
DefaultBackendSvcPort: test.DefaultBeSvcPort,
}

ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, defaultNamer, "" /*kubeSystemUID*/, ctxConfig)
ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, defaultNamer, "" /*kubeSystemUID*/, nil /*IngressMetrics*/, ctxConfig)
fwc := NewFirewallController(ctx, []string{"30000-32767"})
fwc.hasSynced = func() bool { return true }

Expand Down
237 changes: 237 additions & 0 deletions pkg/metrics/features.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,237 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package metrics

import (
"fmt"
"strconv"

"k8s.io/api/networking/v1beta1"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/klog"
)

type feature string

func (f feature) String() string {
return string(f)
}

const (
// WARNING: Please keep the following constants in sync with
// pkg/annotations/ingress.go
// allowHTTPKey tells the Ingress controller to allow/block HTTP access.
allowHTTPKey = "kubernetes.io/ingress.allow-http"
ingressClassKey = "kubernetes.io/ingress.class"
gceIngressClass = "gce"
gceMultiIngressClass = "gce-multi-cluster"
gceL7ILBIngressClass = "gce-internal"
// preSharedCertKey represents the specific pre-shared SSL
// certificate for the Ingress controller to use.
preSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
managedCertKey = "networking.gke.io/managed-certificates"
// staticIPKey is the annotation key used by controller to record GCP static ip.
staticIPKey = "ingress.kubernetes.io/static-ip"

ingress = feature("Ingress")
externalIngress = feature("ExternalIngress")
internalIngress = feature("InternalIngress")
httpEnabled = feature("HTTPEnabled")
hostBasedRouting = feature("HostBasedRouting")
pathBasedRouting = feature("PathBasedRouting")
tlsTermination = feature("TLSTermination")
secretBasedCertsForTLS = feature("SecretBasedCertsForTLS")
preSharedCertsForTLS = feature("PreSharedCertsForTLS")
managedCertsForTLS = feature("ManagedCertsForTLS")
staticGlobalIP = feature("StaticGlobalIP")

servicePort = feature("L7LBServicePort")
externalServicePort = feature("L7XLBServicePort")
internalServicePort = feature("L7ILBServicePort")
neg = feature("NEG")
cloudCDN = feature("CloudCDN")
cloudArmor = feature("CloudArmor")
cloudIAP = feature("CloudIAP")
backendTimeout = feature("BackendTimeout")
backendConnectionDraining = feature("BackendConnectionDraining")
clientIPAffinity = feature("ClientIPAffinity")
cookieAffinity = feature("CookieAffinity")
customRequestHeaders = feature("CustomRequestHeaders")
)

// featuresForIngress returns the list of features for given ingress.
func featuresForIngress(ing *v1beta1.Ingress) []feature {
features := []feature{ingress}

ingKey := fmt.Sprintf("%s/%s", ing.Namespace, ing.Name)
klog.V(4).Infof("Listing features for Ingress %s", ingKey)
ingAnnotations := ing.Annotations

// Determine the type of ingress based on ingress class.
ingClass := ingAnnotations[ingressClassKey]
klog.V(6).Infof("Ingress class value for ingress %s: %s", ingKey, ingClass)
switch ingClass {
case "", gceIngressClass, gceMultiIngressClass:
features = append(features, externalIngress)
case gceL7ILBIngressClass:
features = append(features, internalIngress)
}

// Determine if http is enabled.
if val, ok := ingAnnotations[allowHTTPKey]; !ok {
klog.V(6).Infof("Annotation %s does not exist for ingress %s", allowHTTPKey, ingKey)
features = append(features, httpEnabled)
} else {
klog.V(6).Infof("User specified value for annotation %s on ingress %s: %s", allowHTTPKey, ingKey, val)
v, err := strconv.ParseBool(val)
if err != nil {
klog.Errorf("Failed to parse %s for annotation %s on ingress %s", val, allowHTTPKey, ingKey)
}
if err == nil && v {
features = append(features, httpEnabled)
}
}

// An ingress without a host or http-path is ignored.
hostBased, pathBased := false, false
if len(ing.Spec.Rules) == 0 {
klog.V(6).Infof("Neither host-based nor path-based routing rules are setup for ingress %s", ingKey)
}
for _, rule := range ing.Spec.Rules {
if rule.HTTP != nil && len(rule.HTTP.Paths) > 0 {
klog.V(6).Infof("User specified http paths for ingress %s: %v", ingKey, rule.HTTP.Paths)
pathBased = true
}
if rule.Host != "" {
klog.V(6).Infof("User specified host for ingress %s: %v", ingKey, rule.Host)
hostBased = true
}
if pathBased && hostBased {
break
}
}
if hostBased {
features = append(features, hostBasedRouting)
}
if pathBased {
features = append(features, pathBasedRouting)
}

// SSL certificate based features.
sslConfigured := false
if val, ok := ingAnnotations[preSharedCertKey]; ok {
klog.V(6).Infof("Specified pre-shared certs for ingress %s: %v", ingKey, val)
sslConfigured = true
features = append(features, preSharedCertsForTLS)
}
if val, ok := ingAnnotations[managedCertKey]; ok {
klog.V(6).Infof("Specified google managed certs for ingress %s: %v", ingKey, val)
sslConfigured = true
features = append(features, managedCertsForTLS)
}
if hasSecretBasedCerts(ing) {
sslConfigured = true
features = append(features, secretBasedCertsForTLS)
}
if sslConfigured {
klog.V(6).Infof("TLS termination is configured for ingress %s", ingKey)
features = append(features, tlsTermination)
}

// Both user specified and ingress controller managed global static ips are reported.
if val, ok := ingAnnotations[staticIPKey]; ok && val != "" {
klog.V(6).Infof("Specified static for ingress %s: %s", ingKey, val)
features = append(features, staticGlobalIP)
}
klog.V(4).Infof("Features for ingress %s/%s: %v", ing.Namespace, ing.Name, features)
return features
}

// hasSecretBasedCerts returns true if ingress spec contains a secret based cert.
func hasSecretBasedCerts(ing *v1beta1.Ingress) bool {
for _, tlsSecret := range ing.Spec.TLS {
if tlsSecret.SecretName == "" {
continue
}
klog.V(6).Infof("User specified secret for ingress %s/%s: %s", ing.Namespace, ing.Name, tlsSecret.SecretName)
return true
}
return false
}

// featuresForServicePort returns the list of features for given service port.
func featuresForServicePort(sp utils.ServicePort) []feature {
features := []feature{servicePort}
svcPortKey := newServicePortKey(sp).string()
klog.V(4).Infof("Listing features for service port %s", svcPortKey)
if sp.L7ILBEnabled {
klog.V(6).Infof("L7 ILB is enabled for service port %s", svcPortKey)
features = append(features, internalServicePort)
} else {
features = append(features, externalServicePort)
}
if sp.NEGEnabled {
klog.V(6).Infof("NEG is enabled for service port %s", svcPortKey)
features = append(features, neg)
}
if sp.BackendConfig == nil {
klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features)
return features
}

beConfig := fmt.Sprintf("%s/%s", sp.BackendConfig.Namespace, sp.BackendConfig.Name)
klog.V(6).Infof("Backend config specified for service port %s: %s", svcPortKey, beConfig)

if sp.BackendConfig.Spec.Cdn != nil && sp.BackendConfig.Spec.Cdn.Enabled {
klog.V(6).Infof("Cloud CDN is enabled for service port %s", svcPortKey)
features = append(features, cloudCDN)
}
if sp.BackendConfig.Spec.Iap != nil && sp.BackendConfig.Spec.Iap.Enabled {
klog.V(6).Infof("Cloud IAP is enabled for service port %s", svcPortKey)
features = append(features, cloudIAP)
}
// Possible list of Affinity types:
// NONE, CLIENT_IP, GENERATED_COOKIE, CLIENT_IP_PROTO, or CLIENT_IP_PORT_PROTO.
if sp.BackendConfig.Spec.SessionAffinity != nil {
affinityType := sp.BackendConfig.Spec.SessionAffinity.AffinityType
switch affinityType {
case "GENERATED_COOKIE":
features = append(features, cookieAffinity)
case "CLIENT_IP", "CLIENT_IP_PROTO", "CLIENT_IP_PORT_PROTO":
features = append(features, clientIPAffinity)
}
klog.V(6).Infof("Session affinity %s is configured for service port %s", affinityType, svcPortKey)
}
if sp.BackendConfig.Spec.SecurityPolicy != nil {
klog.V(6).Infof("Security policy %s is configured for service port %s", sp.BackendConfig.Spec.SecurityPolicy, svcPortKey)
features = append(features, cloudArmor)
}
if sp.BackendConfig.Spec.TimeoutSec != nil {
klog.V(6).Infof("Backend timeout(%v secs) is configured for service port %s", sp.BackendConfig.Spec.TimeoutSec, svcPortKey)
features = append(features, backendTimeout)
}
if sp.BackendConfig.Spec.ConnectionDraining != nil {
klog.V(6).Infof("Backend connection draining(%v secs) is configured for service port %s", sp.BackendConfig.Spec.ConnectionDraining.DrainingTimeoutSec, svcPortKey)
features = append(features, backendConnectionDraining)
}
if sp.BackendConfig.Spec.CustomRequestHeaders != nil {
klog.V(6).Infof("Custom request headers configured for service port %s: %v", svcPortKey, sp.BackendConfig.Spec.CustomRequestHeaders.Headers)
features = append(features, customRequestHeaders)
}
klog.V(4).Infof("Features for Service port %s: %v", svcPortKey, features)
return features
}
Loading

0 comments on commit 3fd78ad

Please sign in to comment.