Skip to content

Commit

Permalink
NET-5397 - wire up destination golden tests from sidecar-proxy contro…
Browse files Browse the repository at this point in the history
…ller for xds controller and xdsv2 (#19167)

* NET-5397 - wire up golden tests from sidecar-proxy controller for xds controller and xdsv2

* WIP

* WIP

* everything matching except leafCerts.  need to mock those

* single port destinations working except mixed destinations

* golden test input to xds controller tests for destinations

* proposed fix for failover group naming errors

* clean up test to use helper.

* clean up test to use helper.

* fix test file

* add docstring for test function.

* add docstring for test function.

* fix linting error

* fixing test after route fix merged into main
  • Loading branch information
jmurret authored Oct 24, 2023
1 parent 12ef115 commit 9775758
Show file tree
Hide file tree
Showing 13 changed files with 2,534 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,6 @@ func (b *Builder) buildDestination(
clusterName := fmt.Sprintf("%s.%s", portName, sni)

egName := ""

if details.FailoverConfig != nil {
egName = fmt.Sprintf("%s%d~%s", xdscommon.FailoverClusterNamePrefix, 0, clusterName)
}
Expand Down
159 changes: 159 additions & 0 deletions internal/mesh/internal/controllers/xds/controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,14 @@ import (
"context"
"crypto/x509"
"encoding/pem"
"fmt"
"strings"
"testing"

"github.com/hashicorp/consul/internal/testing/golden"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"google.golang.org/protobuf/encoding/protojson"

svctest "github.com/hashicorp/consul/agent/grpc-external/services/resource/testing"
"github.com/hashicorp/consul/agent/leafcert"
Expand Down Expand Up @@ -998,3 +1002,158 @@ func (suite *xdsControllerTestSuite) TestReconcile_prevWatchesToCancel() {
func TestXdsController(t *testing.T) {
suite.Run(t, new(xdsControllerTestSuite))
}

// TestReconcile_SidecarProxyGoldenFileInputs tests the Reconcile() by using
// the golden test output/expected files from the sidecar proxy tests as inputs
// to the XDS controller reconciliation.
// XDS controller reconciles the full ProxyStateTemplate object. The fields
// that things that it focuses on are leaf certs, endpoints, and trust bundles,
// which is just a subset of the ProxyStateTemplate struct. Prior to XDS controller
// reconciliation, the sidecar proxy controller will have reconciled the other parts
// of the ProxyStateTemplate.
// Since the XDS controller does act on the ProxyStateTemplate, the tests
// utilize that entire object rather than just the parts that XDS controller
// internals reconciles. Namely, by using checking the full ProxyStateTemplate
// rather than just endpoints, leaf certs, and trust bundles, the test also ensures
// side effects or change in scope to XDS controller are not introduce mistakenly.
func (suite *xdsControllerTestSuite) TestReconcile_SidecarProxyGoldenFileInputs() {
path := "../sidecarproxy/builder/testdata"
cases := []string{
// destinations
"destination/l4-single-destination-ip-port-bind-address",
"destination/l4-single-destination-unix-socket-bind-address",
"destination/l4-single-implicit-destination-tproxy",
"destination/l4-multi-destination",
"destination/l4-multiple-implicit-destinations-tproxy",
"destination/l4-implicit-and-explicit-destinations-tproxy",
"destination/mixed-multi-destination",
"destination/multiport-l4-and-l7-multiple-implicit-destinations-tproxy",
"destination/multiport-l4-and-l7-single-implicit-destination-tproxy",
"destination/multiport-l4-and-l7-single-implicit-destination-with-multiple-workloads-tproxy",

//sources

}

for _, name := range cases {
suite.Run(name, func() {
// Create ProxyStateTemplate from the golden file.
pst := JSONToProxyTemplate(suite.T(),
golden.GetBytesAtFilePath(suite.T(), fmt.Sprintf("%s/%s.golden", path, name)))

// Destinations will need endpoint refs set up.
if strings.Split(name, "/")[0] == "destination" && len(pst.ProxyState.Endpoints) == 0 {
suite.addRequiredEndpointsAndRefs(pst)
}

// Store the initial ProxyStateTemplate.
proxyStateTemplate := resourcetest.Resource(pbmesh.ProxyStateTemplateType, "test").
WithData(suite.T(), pst).
Write(suite.T(), suite.client)

// Check with resource service that it exists.
retry.Run(suite.T(), func(r *retry.R) {
suite.client.RequireResourceExists(r, proxyStateTemplate.Id)
})

// Track it in the mapper.
suite.mapper.TrackItem(proxyStateTemplate.Id, []resource.ReferenceOrID{})

// Run the reconcile, and since no ProxyStateTemplate is stored, this simulates a deletion.
err := suite.ctl.Reconcile(context.Background(), suite.runtime, controller.Request{
ID: proxyStateTemplate.Id,
})
require.NoError(suite.T(), err)
require.NotNil(suite.T(), proxyStateTemplate)

// Get the reconciled proxyStateTemplate to check the reconcile results.
reconciledPS := suite.updater.Get(proxyStateTemplate.Id.Name)

// Verify leaf cert contents then hard code them for comparison
// and downstream tests since they change from test run to test run.
require.NotEmpty(suite.T(), reconciledPS.LeafCertificates)
reconciledPS.LeafCertificates = map[string]*pbproxystate.LeafCertificate{
"test-identity": {
Cert: "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n",
Key: "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n",
},
}

// Compare actual vs expected.
actual := prototest.ProtoToJSON(suite.T(), reconciledPS)
expected := golden.Get(suite.T(), actual, name+".golden")
require.JSONEq(suite.T(), expected, actual)
})
}
}

func (suite *xdsControllerTestSuite) addRequiredEndpointsAndRefs(pst *pbmesh.ProxyStateTemplate) {
//get service data
serviceData := &pbcatalog.Service{}
var vp uint32 = 7000
requiredEps := make(map[string]*pbproxystate.EndpointRef)

// iterate through clusters and set up endpoints for cluster/mesh port.
for clusterName := range pst.ProxyState.Clusters {
if clusterName == "null_route_cluster" || clusterName == "original-destination" {
continue
}

//increment the random port number.
vp++
clusterNameSplit := strings.Split(clusterName, ".")
port := clusterNameSplit[0]
svcName := clusterNameSplit[1]

// set up service data with port info.
serviceData.Ports = append(serviceData.Ports, &pbcatalog.ServicePort{
TargetPort: port,
VirtualPort: vp,
Protocol: pbcatalog.Protocol_PROTOCOL_TCP,
})

// create service.
svc := resourcetest.Resource(pbcatalog.ServiceType, svcName).
WithData(suite.T(), &pbcatalog.Service{}).
Write(suite.T(), suite.client)

// create endpoints with svc as owner.
eps := resourcetest.Resource(pbcatalog.ServiceEndpointsType, svcName).
WithData(suite.T(), &pbcatalog.ServiceEndpoints{Endpoints: []*pbcatalog.Endpoint{
{
Ports: map[string]*pbcatalog.WorkloadPort{
"mesh": {
Port: 20000,
Protocol: pbcatalog.Protocol_PROTOCOL_MESH,
},
},
Addresses: []*pbcatalog.WorkloadAddress{
{
Host: "10.1.1.1",
Ports: []string{"mesh"},
},
},
},
}}).
WithOwner(svc.Id).
Write(suite.T(), suite.client)

// add to working list of required endpoints.
requiredEps[clusterName] = &pbproxystate.EndpointRef{
Id: eps.Id,
Port: "mesh",
}
}

// set working list of required endpoints as proxy state's RequiredEndpoints.
pst.RequiredEndpoints = requiredEps
}

func JSONToProxyTemplate(t *testing.T, json []byte) *pbmesh.ProxyStateTemplate {
t.Helper()
proxyTemplate := &pbmesh.ProxyStateTemplate{}
m := protojson.UnmarshalOptions{}
err := m.Unmarshal(json, proxyTemplate)
require.NoError(t, err)
return proxyTemplate
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,182 @@
{
"clusters": {
"original-destination": {
"endpointGroup": {
"passthrough": {
"config": {
"connectTimeout": "5s"
}
}
},
"name": "original-destination"
},
"tcp.api-1.default.dc1.internal.foo.consul": {
"altStatName": "tcp.api-1.default.dc1.internal.foo.consul",
"endpointGroup": {
"dynamic": {
"config": {
"connectTimeout": "5s",
"disablePanicThreshold": true
},
"outboundTls": {
"alpnProtocols": [
"consul~tcp"
],
"outboundMesh": {
"identityKey": "test-identity",
"sni": "api-1.default.dc1.internal.foo.consul",
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api1-identity"
],
"trustBundlePeerNameKey": "local"
}
}
}
}
},
"name": "tcp.api-1.default.dc1.internal.foo.consul"
},
"tcp.api-2.default.dc1.internal.foo.consul": {
"altStatName": "tcp.api-2.default.dc1.internal.foo.consul",
"endpointGroup": {
"dynamic": {
"config": {
"connectTimeout": "5s",
"disablePanicThreshold": true
},
"outboundTls": {
"alpnProtocols": [
"consul~tcp"
],
"outboundMesh": {
"identityKey": "test-identity",
"sni": "api-2.default.dc1.internal.foo.consul",
"validationContext": {
"spiffeIds": [
"spiffe://foo.consul/ap/default/ns/default/identity/api2-identity"
],
"trustBundlePeerNameKey": "local"
}
}
}
}
},
"name": "tcp.api-2.default.dc1.internal.foo.consul"
}
},
"identity": {
"name": "test-identity",
"tenancy": {
"namespace": "default",
"partition": "default",
"peerName": "local"
},
"type": {
"group": "auth",
"groupVersion": "v2beta1",
"kind": "WorkloadIdentity"
}
},
"listeners": [
{
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "1.1.1.1",
"port": 1234
},
"name": "default/local/default/api-1:tcp:1.1.1.1:1234",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-1.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-1.default.default.dc1"
}
}
]
},
{
"capabilities": [
"CAPABILITY_TRANSPARENT"
],
"defaultRouter": {
"l4": {
"cluster": {
"name": "original-destination"
},
"statPrefix": "upstream.original-destination"
}
},
"direction": "DIRECTION_OUTBOUND",
"hostPort": {
"host": "127.0.0.1",
"port": 15001
},
"name": "outbound_listener",
"routers": [
{
"l4": {
"cluster": {
"name": "tcp.api-2.default.dc1.internal.foo.consul"
},
"statPrefix": "upstream.tcp.api-2.default.default.dc1"
},
"match": {
"destinationPort": 7070,
"prefixRanges": [
{
"addressPrefix": "2.2.2.2",
"prefixLen": 32
},
{
"addressPrefix": "3.3.3.3",
"prefixLen": 32
}
]
}
}
]
}
],
"endpoints": {
"tcp.api-1.default.dc1.internal.foo.consul": {
"endpoints": [
{
"healthStatus": "HEALTH_STATUS_HEALTHY",
"hostPort": {
"host": "10.1.1.1",
"port": 20000
}
}
]
},
"tcp.api-2.default.dc1.internal.foo.consul": {
"endpoints": [
{
"healthStatus": "HEALTH_STATUS_HEALTHY",
"hostPort": {
"host": "10.1.1.1",
"port": 20000
}
}
]
}
},
"trustBundles": {
"local": {
"roots": [
"some-root",
"some-other-root"
],
"trustDomain": "some-trust-domain"
}
},
"leafCertificates": {
"test-identity": {
"cert": "-----BEGIN CERTIFICATE-----\nMIICDjCCAbWgAwIBAgIBAjAKBggqhkjOPQQDAjAUMRIwEAYDVQQDEwlUZXN0IENB\nIDEwHhcNMjMxMDE2MTYxMzI5WhcNMjMxMDE2MTYyMzI5WjAAMFkwEwYHKoZIzj0C\nAQYIKoZIzj0DAQcDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9\nta/bGT+5orZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJaOCAQowggEGMA4GA1UdDwEB\n/wQEAwIDuDAdBgNVHSUEFjAUBggrBgEFBQcDAgYIKwYBBQUHAwEwDAYDVR0TAQH/\nBAIwADApBgNVHQ4EIgQg3ogXVz9cqaK2B6xdiJYMa5NtT0KkYv7BA2dR7h9EcwUw\nKwYDVR0jBCQwIoAgq+C1mPlPoGa4lt7sSft1goN5qPGyBIB/3mUHJZKSFY8wbwYD\nVR0RAQH/BGUwY4Zhc3BpZmZlOi8vMTExMTExMTEtMjIyMi0zMzMzLTQ0NDQtNTU1\nNTU1NTU1NTU1LmNvbnN1bC9hcC9kZWZhdWx0L25zL2RlZmF1bHQvaWRlbnRpdHkv\ndGVzdC1pZGVudGl0eTAKBggqhkjOPQQDAgNHADBEAiB6L+t5bzRrBPhiQYNeA7fF\nUCuLWrdjW4Xbv3SLg0IKMgIgfRC5hEx+DqzQxTCP4sexX3hVWMjKoWmHdwiUcg+K\n/IE=\n-----END CERTIFICATE-----\n",
"key": "-----BEGIN EC PRIVATE KEY-----\nMHcCAQEEIFIFkTIL1iUV4O/RpveVHzHs7ZzhSkvYIzbdXDttz9EooAoGCCqGSM49\nAwEHoUQDQgAErErAIosDPheZQGbxFQ4hYC/e9Fi4MG9z/zjfCnCq/oK9ta/bGT+5\norZqTmdN/ICsKQDhykxZ2u/Xr6845zhcJQ==\n-----END EC PRIVATE KEY-----\n"
}
}
}
Loading

0 comments on commit 9775758

Please sign in to comment.