Skip to content

Commit

Permalink
[1.17.0 backport] [NET-6305] xds: Ensure v2 route match and protocol …
Browse files Browse the repository at this point in the history 
…are populated for gRPC (#19367)

* backport of commit 9775758

* backport of commit 59d4962

* backport of commit abbf858

* backport of commit dc00be0

* backport of commit a7803bd

---------

Co-authored-by: John Murret <john.murret@hashicorp.com>
  • Loading branch information
@zalimeni @jmurret
zalimeni and jmurret authored Oct 25, 2023
1 parent 7ff7649 commit 312a38e
Show file tree
Hide file tree
Showing 149 changed files with 11,812 additions and 865 deletions.
18 changes: 14 additions & 4 deletions agent/xds/proxystateconverter/clusters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -462,7 +462,7 @@ func (s *Converter) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathP
if protocol == "" {
protocol = cfg.Protocol
}
namedCluster.cluster.Protocol = protocol
namedCluster.cluster.Protocol = protocolMap[protocol]
if cfg.MaxInboundConnections > 0 {
namedCluster.cluster.GetEndpointGroup().GetStatic().GetConfig().
CircuitBreakers = &pbproxystate.CircuitBreakers{
Expand Down Expand Up @@ -646,7 +646,7 @@ func (s *Converter) makeUpstreamClusterForPreparedQuery(upstream structs.Upstrea

if c == nil {
c = &pbproxystate.Cluster{
Protocol: cfg.Protocol,
Protocol: protocolMap[cfg.Protocol],
Group: &pbproxystate.Cluster_EndpointGroup{
EndpointGroup: &pbproxystate.EndpointGroup{
Group: &pbproxystate.EndpointGroup_Dynamic{
Expand Down Expand Up @@ -932,7 +932,7 @@ func (s *Converter) makeUpstreamClustersForDiscoveryChain(
} else {
cluster := &pbproxystate.Cluster{
AltStatName: mappedTargets.baseClusterName,
Protocol: upstreamConfig.Protocol,
Protocol: protocolMap[upstreamConfig.Protocol],
Group: &pbproxystate.Cluster_EndpointGroup{
EndpointGroup: &pbproxystate.EndpointGroup{
Group: &pbproxystate.EndpointGroup_Dynamic{
Expand All @@ -952,7 +952,7 @@ func (s *Converter) makeUpstreamClustersForDiscoveryChain(
failoverGroup.EndpointGroups = endpointGroups
cluster := &pbproxystate.Cluster{
AltStatName: mappedTargets.baseClusterName,
Protocol: upstreamConfig.Protocol,
Protocol: protocolMap[upstreamConfig.Protocol],
Group: &pbproxystate.Cluster_FailoverGroup{
FailoverGroup: failoverGroup,
},
Expand Down Expand Up @@ -1251,3 +1251,13 @@ func makeOutlierDetection(p *structs.PassiveHealthCheck, override *structs.Passi

return od
}

// protocolMap converts config entry protocols to proxystate protocol values.
// As documented on config entry protos, the valid values are "tcp", "http",
// "http2" and "grpc". Anything else is treated as tcp.
var protocolMap = map[string]pbproxystate.Protocol{
"http": pbproxystate.Protocol_PROTOCOL_HTTP,
"http2": pbproxystate.Protocol_PROTOCOL_HTTP2,
"grpc": pbproxystate.Protocol_PROTOCOL_GRPC,
"tcp": pbproxystate.Protocol_PROTOCOL_TCP,
}
20 changes: 10 additions & 10 deletions agent/xdsv2/cluster_resources.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) {
return clusters, nil
}

func (pr *ProxyResources) makeEnvoyCluster(name string, protocol string, eg *pbproxystate.EndpointGroup) (*envoy_cluster_v3.Cluster, error) {
func (pr *ProxyResources) makeEnvoyCluster(name string, protocol pbproxystate.Protocol, eg *pbproxystate.EndpointGroup) (*envoy_cluster_v3.Cluster, error) {
if eg != nil {
switch t := eg.Group.(type) {
case *pbproxystate.EndpointGroup_Dynamic:
Expand All @@ -103,7 +103,7 @@ func (pr *ProxyResources) makeEnvoyCluster(name string, protocol string, eg *pbp
return nil, fmt.Errorf("no endpoint group")
}

func (pr *ProxyResources) makeEnvoyDynamicCluster(name string, protocol string, dynamic *pbproxystate.DynamicEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
func (pr *ProxyResources) makeEnvoyDynamicCluster(name string, protocol pbproxystate.Protocol, dynamic *pbproxystate.DynamicEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
cluster := &envoy_cluster_v3.Cluster{
Name: name,
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS},
Expand Down Expand Up @@ -153,7 +153,7 @@ func (pr *ProxyResources) makeEnvoyDynamicCluster(name string, protocol string,

}

func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, static *pbproxystate.StaticEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol pbproxystate.Protocol, static *pbproxystate.StaticEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
cluster := &envoy_cluster_v3.Cluster{
Name: name,
ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC},
Expand Down Expand Up @@ -182,11 +182,11 @@ func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, s
return cluster, nil
}

func (pr *ProxyResources) makeEnvoyDnsCluster(name string, protocol string, dns *pbproxystate.DNSEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
func (pr *ProxyResources) makeEnvoyDnsCluster(name string, protocol pbproxystate.Protocol, dns *pbproxystate.DNSEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
return nil, nil
}

func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol string, passthrough *pbproxystate.PassthroughEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol pbproxystate.Protocol, passthrough *pbproxystate.PassthroughEndpointGroup) (*envoy_cluster_v3.Cluster, error) {
cluster := &envoy_cluster_v3.Cluster{
Name: name,
ConnectTimeout: passthrough.Config.ConnectTimeout,
Expand All @@ -207,7 +207,7 @@ func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol stri
return cluster, nil
}

func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol string, fg *pbproxystate.FailoverGroup) ([]*envoy_cluster_v3.Cluster, error) {
func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol pbproxystate.Protocol, fg *pbproxystate.FailoverGroup) ([]*envoy_cluster_v3.Cluster, error) {
var clusters []*envoy_cluster_v3.Cluster
if fg != nil {
var egNames []string
Expand Down Expand Up @@ -250,8 +250,8 @@ func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol string
return clusters, nil
}

func addLocalAppHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error {
if !(protocol == "http2" || protocol == "grpc") {
func addLocalAppHttpProtocolOptions(protocol pbproxystate.Protocol, c *envoy_cluster_v3.Cluster) error {
if !(protocol == pbproxystate.Protocol_PROTOCOL_HTTP2 || protocol == pbproxystate.Protocol_PROTOCOL_GRPC) {
// do not error. returning nil means it won't get set.
return nil
}
Expand All @@ -274,8 +274,8 @@ func addLocalAppHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster
return nil
}

func addHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error {
if !(protocol == "http2" || protocol == "grpc") {
func addHttpProtocolOptions(protocol pbproxystate.Protocol, c *envoy_cluster_v3.Cluster) error {
if !(protocol == pbproxystate.Protocol_PROTOCOL_HTTP2 || protocol == pbproxystate.Protocol_PROTOCOL_GRPC) {
// do not error. returning nil means it won't get set.
return nil
}
Expand Down
177 changes: 107 additions & 70 deletions agent/xdsv2/resources_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@
package xdsv2

import (
"os"
"path/filepath"
"fmt"
envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
envoy_tls_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
"github.com/hashicorp/consul/internal/testing/golden"
"sort"
"testing"

Expand All @@ -23,62 +26,110 @@ import (
"google.golang.org/protobuf/proto"
)

func TestResources_ImplicitDestinations(t *testing.T) {
var testTypeUrlToPrettyName = map[string]string{
xdscommon.ListenerType: "listeners",
xdscommon.RouteType: "routes",
xdscommon.ClusterType: "clusters",
xdscommon.EndpointType: "endpoints",
xdscommon.SecretType: "secrets",
}

cases := map[string]struct {
}{
"l4-single-implicit-destination-tproxy": {},
// TestAllResourcesFromIR_XDSGoldenFileInputs tests the AllResourcesFromIR() by
// using the golden test output/expected files from the XDS controller tests as
// inputs to the XDSV2 resources generation.
func TestAllResourcesFromIR_XDSGoldenFileInputs(t *testing.T) {
inputPath := "../../internal/mesh/internal/controllers/xds"

cases := []string{
// destinations - please add in alphabetical order
"destination/l4-single-destination-ip-port-bind-address",
"destination/l4-single-destination-unix-socket-bind-address",
"destination/l4-single-implicit-destination-tproxy",
"destination/l4-multi-destination",
"destination/l4-multiple-implicit-destinations-tproxy",
"destination/l4-implicit-and-explicit-destinations-tproxy",
"destination/mixed-multi-destination",
"destination/multiport-l4-and-l7-multiple-implicit-destinations-tproxy",
"destination/multiport-l4-and-l7-single-implicit-destination-tproxy",
"destination/multiport-l4-and-l7-single-implicit-destination-with-multiple-workloads-tproxy",

//sources - please add in alphabetical order
"source/l4-multiple-workload-addresses-with-specific-ports",
"source/l4-multiple-workload-addresses-without-ports",
"source/l4-single-workload-address-without-ports",
"source/l7-expose-paths",
"source/local-and-inbound-connections",
"source/multiport-l4-multiple-workload-addresses-with-specific-ports",
"source/multiport-l4-multiple-workload-addresses-without-ports",
"source/multiport-l4-workload-with-only-mesh-port",
"source/multiport-l7-multiple-workload-addresses-with-specific-ports",
"source/multiport-l7-multiple-workload-addresses-without-ports",
}

for name := range cases {
goldenValueInput := goldenValueJSON(t, name, "input")

proxyTemplate := jsonToProxyTemplate(t, goldenValueInput)
generator := NewResourceGenerator(testutil.Logger(t))

resources, err := generator.AllResourcesFromIR(&proxytracker.ProxyState{ProxyState: proxyTemplate.ProxyState})
require.NoError(t, err)

verifyClusterResourcesToGolden(t, resources, name)
verifyListenerResourcesToGolden(t, resources, name)

for _, name := range cases {
t.Run(name, func(t *testing.T) {
// Arrange - paths to input and output golden files.
testFile := fmt.Sprintf("%s.golden", name)
inputFilePath := fmt.Sprintf("%s/testdata/%s", inputPath, testFile)
inputValueInput := golden.GetBytesAtFilePath(t, inputFilePath)

// Act.
ps := jsonToProxyState(t, inputValueInput)
generator := NewResourceGenerator(testutil.Logger(t))
resources, err := generator.AllResourcesFromIR(&proxytracker.ProxyState{ProxyState: ps})
require.NoError(t, err)

// Assert.
// Assert all resources were generated.
typeUrls := []string{
xdscommon.ListenerType,
xdscommon.RouteType,
xdscommon.ClusterType,
xdscommon.EndpointType,
// TODO(proxystate): add in future
//xdscommon.SecretType,
}
require.Len(t, resources, len(typeUrls))

// Assert each resource type has actual XDS matching expected XDS.
for _, typeUrl := range typeUrls {
prettyName := testTypeUrlToPrettyName[typeUrl]
t.Run(prettyName, func(t *testing.T) {
items, ok := resources[typeUrl]
require.True(t, ok)

// sort resources so they don't show up as flakey tests as
// ordering in JSON is not guaranteed.
sort.Slice(items, func(i, j int) bool {
switch typeUrl {
case xdscommon.ListenerType:
return items[i].(*envoy_listener_v3.Listener).Name < items[j].(*envoy_listener_v3.Listener).Name
case xdscommon.RouteType:
return items[i].(*envoy_route_v3.RouteConfiguration).Name < items[j].(*envoy_route_v3.RouteConfiguration).Name
case xdscommon.ClusterType:
return items[i].(*envoy_cluster_v3.Cluster).Name < items[j].(*envoy_cluster_v3.Cluster).Name
case xdscommon.EndpointType:
return items[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < items[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName
case xdscommon.SecretType:
return items[i].(*envoy_tls_v3.Secret).Name < items[j].(*envoy_tls_v3.Secret).Name
default:
panic("not possible")
}
})

// Compare actual to expected.
resp, err := response.CreateResponse(typeUrl, "00000001", "00000001", items)
require.NoError(t, err)
gotJSON := protoToJSON(t, resp)

expectedJSON := golden.Get(t, gotJSON, fmt.Sprintf("%s/%s", prettyName, testFile))
require.JSONEq(t, expectedJSON, gotJSON)
})
}
})
}
}

func verifyClusterResourcesToGolden(t *testing.T, resources map[string][]proto.Message, testName string) {
clusters := resources[xdscommon.ClusterType]

// The order of clusters returned via CDS isn't relevant, so it's safe
// to sort these for the purposes of test comparisons.
sort.Slice(clusters, func(i, j int) bool {
return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name
})

resp, err := response.CreateResponse(xdscommon.ClusterType, "00000001", "00000001", clusters)
require.NoError(t, err)
gotJSON := protoToJSON(t, resp)

expectedJSON := goldenValue(t, filepath.Join("clusters", testName), "output")
require.JSONEq(t, expectedJSON, gotJSON)
}

func verifyListenerResourcesToGolden(t *testing.T, resources map[string][]proto.Message, testName string) {
listeners := resources[xdscommon.ListenerType]

// The order of clusters returned via CDS isn't relevant, so it's safe
// to sort these for the purposes of test comparisons.
sort.Slice(listeners, func(i, j int) bool {
return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name
})

resp, err := response.CreateResponse(xdscommon.ListenerType, "00000001", "00000001", listeners)
require.NoError(t, err)
gotJSON := protoToJSON(t, resp)

expectedJSON := goldenValue(t, filepath.Join("listeners", testName), "output")
require.JSONEq(t, expectedJSON, gotJSON)
}

func protoToJSON(t *testing.T, pb proto.Message) string {
t.Helper()
m := protojson.MarshalOptions{
Expand All @@ -89,25 +140,11 @@ func protoToJSON(t *testing.T, pb proto.Message) string {
return string(gotJSON)
}

func jsonToProxyTemplate(t *testing.T, json []byte) *meshv2beta1.ProxyStateTemplate {
func jsonToProxyState(t *testing.T, json []byte) *meshv2beta1.ProxyState {
t.Helper()
um := protojson.UnmarshalOptions{}
proxyTemplate := &meshv2beta1.ProxyStateTemplate{}
err := um.Unmarshal(json, proxyTemplate)
require.NoError(t, err)
return proxyTemplate
}

func goldenValueJSON(t *testing.T, goldenFile, inputOutput string) []byte {
t.Helper()
goldenPath := filepath.Join("testdata", inputOutput, goldenFile) + ".golden"

content, err := os.ReadFile(goldenPath)
ps := &meshv2beta1.ProxyState{}
err := um.Unmarshal(json, ps)
require.NoError(t, err)
return content
}

func goldenValue(t *testing.T, goldenFile, inputOutput string) string {
t.Helper()
return string(goldenValueJSON(t, goldenFile, inputOutput))
return ps
}
Loading

0 comments on commit 312a38e

Please sign in to comment.