Skip to content

Commit

Permalink
Add Ingress usage metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
skmatti committed Dec 24, 2019
1 parent 1d46157 commit 532b38f
Show file tree
Hide file tree
Showing 14 changed files with 1,131 additions and 9 deletions.
9 changes: 8 additions & 1 deletion cmd/glbc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (

ingctx "k8s.io/ingress-gce/pkg/context"
"k8s.io/ingress-gce/pkg/controller"
"k8s.io/ingress-gce/pkg/metrics"
"k8s.io/ingress-gce/pkg/neg"
negtypes "k8s.io/ingress-gce/pkg/neg/types"

Expand Down Expand Up @@ -131,6 +132,9 @@ func main() {
}
kubeSystemUID := kubeSystemNS.GetUID()

// Initialize ingress usage metrics.
ingresMetrics := metrics.NewIngressMetrics()

cloud := app.NewGCEClient()
defaultBackendServicePort := app.DefaultBackendServicePort(kubeClient)
ctxConfig := ingctx.ControllerContextConfig{
Expand All @@ -144,7 +148,7 @@ func main() {
ASMConfigMapNamespace: flags.F.ASMConfigMapBasedConfigNamespace,
ASMConfigMapName: flags.F.ASMConfigMapBasedConfigCMName,
}
ctx := ingctx.NewControllerContext(kubeConfig, kubeClient, backendConfigClient, frontendConfigClient, cloud, namer, kubeSystemUID, ctxConfig)
ctx := ingctx.NewControllerContext(kubeConfig, kubeClient, backendConfigClient, frontendConfigClient, cloud, namer, kubeSystemUID, ingresMetrics, ctxConfig)
go app.RunHTTPServer(ctx.HealthCheck)

if !flags.F.LeaderElection.LeaderElect {
Expand Down Expand Up @@ -229,6 +233,9 @@ func runControllers(ctx *ingctx.ControllerContext) {
go fwc.Run()
klog.V(0).Infof("firewall controller started")

// Export ingress usage metrics.
go ctx.IngressMetrics.Run(stopCh)

ctx.Start(stopCh)
lbc.Init()
lbc.Run()
Expand Down
5 changes: 5 additions & 0 deletions pkg/context/context.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
"k8s.io/ingress-gce/pkg/common/typed"
frontendconfigclient "k8s.io/ingress-gce/pkg/frontendconfig/client/clientset/versioned"
informerfrontendconfig "k8s.io/ingress-gce/pkg/frontendconfig/client/informers/externalversions/frontendconfig/v1beta1"
"k8s.io/ingress-gce/pkg/metrics"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/ingress-gce/pkg/utils/namer"
"k8s.io/klog"
Expand Down Expand Up @@ -75,6 +76,8 @@ type ControllerContext struct {
DestinationRuleInformer cache.SharedIndexInformer
ConfigMapInformer cache.SharedIndexInformer

IngressMetrics *metrics.IngressMetrics

healthChecks map[string]func() error

lock sync.Mutex
Expand Down Expand Up @@ -106,6 +109,7 @@ func NewControllerContext(
cloud *gce.Cloud,
namer *namer.Namer,
kubeSystemUID types.UID,
ingressMetrics *metrics.IngressMetrics,
config ControllerContextConfig) *ControllerContext {

context := &ControllerContext{
Expand All @@ -114,6 +118,7 @@ func NewControllerContext(
Cloud: cloud,
ClusterNamer: namer,
KubeSystemUID: kubeSystemUID,
IngressMetrics: ingressMetrics,
ControllerContextConfig: config,
IngressInformer: informerv1beta1.NewIngressInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
ServiceInformer: informerv1.NewServiceInformer(kubeClient, config.Namespace, config.ResyncPeriod, utils.NewNamespaceIndexer()),
Expand Down
8 changes: 8 additions & 0 deletions pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import (
"k8s.io/ingress-gce/pkg/healthchecks"
"k8s.io/ingress-gce/pkg/instances"
"k8s.io/ingress-gce/pkg/loadbalancers"
"k8s.io/ingress-gce/pkg/metrics"
negtypes "k8s.io/ingress-gce/pkg/neg/types"
ingsync "k8s.io/ingress-gce/pkg/sync"
"k8s.io/ingress-gce/pkg/tls"
Expand Down Expand Up @@ -531,6 +532,10 @@ func (lbc *LoadBalancerController) sync(key string) error {
if err != nil && ingExists {
lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "GC", fmt.Sprintf("Error during GC: %v", err))
}
// Delete the ingress state for metrics after GC is successful.
if err == nil && ingExists {
lbc.ctx.IngressMetrics.Delete(key)
}
return err
}

Expand All @@ -555,6 +560,9 @@ func (lbc *LoadBalancerController) sync(key string) error {
syncErr := lbc.ingSyncer.Sync(syncState)
if syncErr != nil {
lbc.ctx.Recorder(ing.Namespace).Eventf(ing, apiv1.EventTypeWarning, "Sync", fmt.Sprintf("Error during sync: %v", syncErr.Error()))
} else {
// Insert/update the ingress state for metrics after successful sync.
lbc.ctx.IngressMetrics.Set(key, metrics.NewIngressState(ing, urlMap.AllServicePorts()))
}

// Garbage collection will occur regardless of an error occurring. If an error occurred,
Expand Down
3 changes: 2 additions & 1 deletion pkg/controller/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
"k8s.io/ingress-gce/pkg/flags"
"k8s.io/ingress-gce/pkg/instances"
"k8s.io/ingress-gce/pkg/loadbalancers"
"k8s.io/ingress-gce/pkg/metrics"
"k8s.io/ingress-gce/pkg/test"
"k8s.io/ingress-gce/pkg/tls"
"k8s.io/ingress-gce/pkg/utils"
Expand Down Expand Up @@ -72,7 +73,7 @@ func newLoadBalancerController() *LoadBalancerController {
HealthCheckPath: "/",
DefaultBackendHealthCheckPath: "/healthz",
}
ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, namer, "" /*kubeSystemUID*/, ctxConfig)
ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, namer, "" /*kubeSystemUID*/, metrics.NewIngressMetrics(), ctxConfig)
lbc := NewLoadBalancerController(ctx, stopCh)
// TODO(rramkumar): Fix this so we don't have to override with our fake
lbc.instancePool = instances.NewNodePool(instances.NewFakeInstanceGroups(sets.NewString(), namer), namer)
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/translator/translator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func fakeTranslator() *Translator {
HealthCheckPath: "/",
DefaultBackendHealthCheckPath: "/healthz",
}
ctx := context.NewControllerContext(nil, client, backendConfigClient, nil, nil, defaultNamer, "" /*kubeSystemUID*/, ctxConfig)
ctx := context.NewControllerContext(nil, client, backendConfigClient, nil, nil, defaultNamer, "" /*kubeSystemUID*/, nil /*IngressMetrics*/, ctxConfig)
gce := &Translator{
ctx: ctx,
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/firewalls/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ func newFirewallController() *FirewallController {
DefaultBackendSvcPort: test.DefaultBeSvcPort,
}

ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, defaultNamer, "" /*kubeSystemUID*/, ctxConfig)
ctx := context.NewControllerContext(nil, kubeClient, backendConfigClient, nil, fakeGCE, defaultNamer, "" /*kubeSystemUID*/, nil /*IngressMetrics*/, ctxConfig)
fwc := NewFirewallController(ctx, []string{"30000-32767"})
fwc.hasSynced = func() bool { return true }

Expand Down
185 changes: 185 additions & 0 deletions pkg/metrics/features.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
package metrics

import (
"strconv"

"k8s.io/api/networking/v1beta1"
"k8s.io/ingress-gce/pkg/utils"
"k8s.io/klog"
)

type Feature string

func (f Feature) String() string {
return string(f)
}

const (
// allowHTTPKey tells the Ingress controller to allow/block HTTP access.
allowHTTPKey = "kubernetes.io/ingress.allow-http"
ingressClassKey = "kubernetes.io/ingress.class"
gceIngressClass = "gce"
gceMultiIngressClass = "gce-multi-cluster"
gceL7ILBIngressClass = "gce-internal"
// preSharedCertKey represents the specific pre-shared SSL
// certificate for the Ingress controller to use.
preSharedCertKey = "ingress.gcp.kubernetes.io/pre-shared-cert"
managedCertKey = "networking.gke.io/managed-certificates"
// staticIPKey is the annotation key used by controller to record GCP static ip.
staticIPKey = "ingress.kubernetes.io/static-ip"

ingress = Feature("Ingress")
externalIngress = Feature("ExternalIngress")
internalIngress = Feature("InternalIngress")
httpEnabled = Feature("HTTPEnabled")
hostBasedRouting = Feature("HostBasedRouting")
pathBasedRouting = Feature("PathBasedRouting")
tlsTermination = Feature("TLSTermination")
secretBasedCertsForTLS = Feature("SecretBasedCertsForTLS")
preSharedCertsForTLS = Feature("PreSharedCertsForTLS")
managedCertsForTLS = Feature("ManagedCertsForTLS")
staticGlobalIP = Feature("StaticGlobalIP")

servicePort = Feature("L7LBServicePort")
externalServicePort = Feature("L7XLBServicePort")
internalServicePort = Feature("L7ILBServicePort")
neg = Feature("NEG")
cloudCDN = Feature("CloudCDN")
cloudArmor = Feature("CloudArmor")
cloudIAP = Feature("CloudIAP")
backendTimeout = Feature("BackendTimeout")
backendConnectionDraining = Feature("BackendConnectionDraining")
clientIPAffinity = Feature("ClientIPAffinity")
cookieAffinity = Feature("CookieAffinity")
customRequestHeaders = Feature("CustomRequestHeaders")
)

// FeaturesForIngress returns the list of features for given ingress.
func FeaturesForIngress(ing *v1beta1.Ingress) []Feature {
features := []Feature{ingress}

klog.V(4).Infof("Listing features for Ingress %s/%s", ing.Namespace, ing.Name)
ingAnnotations := ing.Annotations

// Determine the type of ingress based on ingress class.
switch ingAnnotations[ingressClassKey] {
case "", gceIngressClass, gceMultiIngressClass:
features = append(features, externalIngress)
case gceL7ILBIngressClass:
features = append(features, internalIngress)
}

// Determine if http is enabled.
if val, ok := ingAnnotations[allowHTTPKey]; !ok {
features = append(features, httpEnabled)
} else {
v, err := strconv.ParseBool(val)
if err == nil && v {
features = append(features, httpEnabled)
}
}

// An ingress without a host or http-path is ignored.
hostBased, pathBased := false, false
for _, rule := range ing.Spec.Rules {
if rule.HTTP != nil && len(rule.HTTP.Paths) > 0 {
pathBased = true
}
if rule.Host != "" {
hostBased = true
}
if pathBased && hostBased {
break
}
}
if hostBased {
features = append(features, hostBasedRouting)
}
if pathBased {
features = append(features, pathBasedRouting)
}

// SSL certificate based features.
sslConfigured := false
if _, ok := ingAnnotations[preSharedCertKey]; ok {
sslConfigured = true
features = append(features, preSharedCertsForTLS)
}
if _, ok := ingAnnotations[managedCertKey]; ok {
sslConfigured = true
features = append(features, managedCertsForTLS)
}
if hasSecretBasedCerts(ing) {
sslConfigured = true
features = append(features, secretBasedCertsForTLS)
}
if sslConfigured {
features = append(features, tlsTermination)
}

// Both user specified and ingress controller managed global static ips are reported.
if val, ok := ingAnnotations[staticIPKey]; ok && val != "" {
features = append(features, staticGlobalIP)
}
klog.V(4).Infof("Features for ingress %s/%s are %v", ing.Namespace, ing.Name, features)
return features
}

func hasSecretBasedCerts(ing *v1beta1.Ingress) bool {
for _, tlsSecret := range ing.Spec.TLS {
if tlsSecret.SecretName == "" {
continue
}
return true
}
return false
}

// FeaturesForServicePort returns the list of features for given service port.
func FeaturesForServicePort(sp utils.ServicePort) []Feature {
features := []Feature{servicePort}
klog.V(4).Infof("Listing features for service port %#v", sp)
if sp.L7ILBEnabled {
features = append(features, internalServicePort)
} else {
features = append(features, externalServicePort)
}
if sp.NEGEnabled {
features = append(features, neg)
}
if sp.BackendConfig == nil {
klog.V(4).Infof("Features for Service port %v are %v", sp.ID, features)
return features
}

if sp.BackendConfig.Spec.Cdn != nil && sp.BackendConfig.Spec.Cdn.Enabled {
features = append(features, cloudCDN)
}
if sp.BackendConfig.Spec.Iap != nil && sp.BackendConfig.Spec.Iap.Enabled {
features = append(features, cloudIAP)
}
// Possible list of Affinity types:
// NONE, CLIENT_IP, GENERATED_COOKIE, CLIENT_IP_PROTO, or CLIENT_IP_PORT_PROTO.
if sp.BackendConfig.Spec.SessionAffinity != nil {
switch sp.BackendConfig.Spec.SessionAffinity.AffinityType {
case "GENERATED_COOKIE":
features = append(features, cookieAffinity)
case "CLIENT_IP", "CLIENT_IP_PROTO", "CLIENT_IP_PORT_PROTO":
features = append(features, clientIPAffinity)
}
}
if sp.BackendConfig.Spec.SecurityPolicy != nil {
features = append(features, cloudArmor)
}
if sp.BackendConfig.Spec.TimeoutSec != nil {
features = append(features, backendTimeout)
}
if sp.BackendConfig.Spec.ConnectionDraining != nil {
features = append(features, backendConnectionDraining)
}
if sp.BackendConfig.Spec.CustomRequestHeaders != nil {
features = append(features, customRequestHeaders)
}
klog.V(4).Infof("Features for Service port %v are %v", sp.ID, features)
return features
}
Loading

0 comments on commit 532b38f

Please sign in to comment.