Merge branch 'main' into doc-update-for-log-file-changes

.changelog/17936.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+acl: Add new `acl.tokens.dns` config field which specifies the token used implicitly during dns checks.
+```

.github/scripts/filter_changed_files_go_test.sh

@@ -2,36 +2,41 @@
 # Copyright (c) HashiCorp, Inc.
 # SPDX-License-Identifier: BUSL-1.1
 
+set -euo pipefail
 
 # Get the list of changed files
-files_to_check=$(git diff --name-only origin/$GITHUB_BASE_REF)
+# Using `git merge-base` ensures that we're always comparing against the correct branch point. 
+#For example, given the commits:
+#
+# A---B---C---D---W---X---Y---Z # origin/main
+#             \---E---F         # feature/branch
+#
+# ... `git merge-base origin/$SKIP_CHECK_BRANCH HEAD` would return commit `D`
+# `...HEAD` specifies from the common ancestor to the latest commit on the current branch (HEAD)..
+files_to_check=$(git diff --name-only "$(git merge-base origin/$SKIP_CHECK_BRANCH HEAD~)"...HEAD)
 
 # Define the directories to check
 skipped_directories=("docs/" "ui/" "website/" "grafana/")
 
-# Initialize a variable to track directories outside the skipped ones
-other_directories=""
-trigger_ci=true
+# Loop through the changed files and find directories/files outside the skipped ones
+for file_to_check in "${files_to_check[@]}"; do
+	file_is_skipped=false
+	for dir in "${skipped_directories[@]}"; do
+		if [[ "$file_to_check" == "$dir"* ]] || [[ "$file_to_check" == *.md && "$dir" == *"/" ]]; then
+			file_is_skipped=true
+			break
+		fi
+	done
+	if [ "$file_is_skipped" != "true" ]; then
+		echo -e $file_to_check
+        SKIP_CI=false
+		echo "Changes detected in non-documentation files - skip-ci: $SKIP_CI"
+        echo "skip-ci=$SKIP_CI" >> "$GITHUB_OUTPUT"
+		exit 0 ## if file is outside of the skipped_directory exit script
+	fi
+done
 
-# # Loop through the changed files and find directories/files outside the skipped ones
-# for file_to_check in $files_to_check; do
-# 	file_is_skipped=false
-# 	for dir in "${skipped_directories[@]}"; do
-# 		if [[ "$file_to_check" == "$dir"* ]] || [[ "$file_to_check" == *.md && "$dir" == *"/" ]]; then
-# 			file_is_skipped=true
-# 			break
-# 		fi
-# 	done
-# 	if [ "$file_is_skipped" = "false" ]; then
-# 		other_directories+="$(dirname "$file_to_check")\n"
-# 		trigger_ci=true
-# 		echo "Non doc file(s) changed - triggered ci: $trigger_ci"
-# 		echo -e $other_directories
-# 		echo "trigger-ci=$trigger_ci" >>"$GITHUB_OUTPUT"
-# 		exit 0 ## if file is outside of the skipped_directory exit script
-# 	fi
-# done
-
-# echo "Only doc file(s) changed - triggered ci: $trigger_ci"
-echo "Doc file(s) change detection is currently disabled - triggering ci"
-echo "trigger-ci=$trigger_ci" >>"$GITHUB_OUTPUT"
+echo -e "$files_to_check"
+SKIP_CI=true
+echo "Changes detected in only documentation files - skip-ci: $SKIP_CI"
+echo "skip-ci=$SKIP_CI" >> "$GITHUB_OUTPUT"

.github/workflows/go-tests.yml

@@ -22,6 +22,7 @@ permissions:
 env:
   TEST_RESULTS: /tmp/test-results
   GOPRIVATE: github.com/hashicorp # Required for enterprise deps
+  SKIP_CHECK_BRANCH: ${{ github.head_ref || github.ref_name }}
 
 # concurrency
 concurrency:
@@ -33,7 +34,7 @@ jobs:
     runs-on: ubuntu-latest 
     name: Get files changed and conditionally skip CI
     outputs:
-      trigger-ci: ${{ steps.read-files.outputs.trigger-ci }}
+      skip-ci: ${{ steps.read-files.outputs.skip-ci }}
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
         with:
@@ -45,7 +46,7 @@ jobs:
   setup:
     needs: [conditional-skip]
     name: Setup
-    if: needs.conditional-skip.outputs.trigger-ci == 'true'
+    if: needs.conditional-skip.outputs.skip-ci != 'true'
     runs-on: ubuntu-latest
     outputs:
       compute-small: ${{ steps.setup-outputs.outputs.compute-small }}
@@ -506,7 +507,7 @@ jobs:
     - go-test-32bit
     # - go-test-s390x
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
-    if: always() && needs.conditional-skip.outputs.trigger-ci == 'true'
+    if: always() && needs.conditional-skip.outputs.skip-ci != 'true'
     steps:
       - name: evaluate upstream job results
         run: |

.github/workflows/test-integrations.yml

@@ -24,6 +24,7 @@ env:
   # strip the hashicorp/ off the front of github.repository for consul
   CONSUL_LATEST_IMAGE_NAME: ${{ endsWith(github.repository, '-enterprise') && github.repository || 'hashicorp/consul' }}
   GOPRIVATE: github.com/hashicorp # Required for enterprise deps
+  SKIP_CHECK_BRANCH: ${{ github.head_ref || github.ref_name }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
@@ -34,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest 
     name: Get files changed and conditionally skip CI
     outputs:
-      trigger-ci: ${{ steps.read-files.outputs.trigger-ci }}
+      skip-ci: ${{ steps.read-files.outputs.skip-ci }}
     steps:
       - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
         with:
@@ -47,7 +48,7 @@ jobs:
     needs: [conditional-skip]
     runs-on: ubuntu-latest
     name: Setup
-    if: needs.conditional-skip.outputs.trigger-ci == 'true'
+    if: needs.conditional-skip.outputs.skip-ci != 'true'
     outputs:
       compute-small: ${{ steps.runners.outputs.compute-small }}
       compute-medium: ${{ steps.runners.outputs.compute-medium }}
@@ -495,7 +496,7 @@ jobs:
     - envoy-integration-test
     - compatibility-integration-test
     runs-on: ${{ fromJSON(needs.setup.outputs.compute-small) }}
-    if: always() && needs.conditional-skip.outputs.trigger-ci == 'true'
+    if: always() && needs.conditional-skip.outputs.skip-ci != 'true'
     steps:
       - name: evaluate upstream job results
         run: |

CHANGELOG.md

@@ -1,3 +1,98 @@
+## 1.16.2 (September 19, 2023)
+
+SECURITY:
+
+* Upgrade to use Go 1.20.8. This resolves CVEs
+[CVE-2023-39320](https://github.com/advisories/GHSA-rxv8-v965-v333) (`cmd/go`),
+[CVE-2023-39318](https://github.com/advisories/GHSA-vq7j-gx56-rxjh) (`html/template`),
+[CVE-2023-39319](https://github.com/advisories/GHSA-vv9m-32rr-3g55) (`html/template`),
+[CVE-2023-39321](https://github.com/advisories/GHSA-9v7r-x7cv-v437) (`crypto/tls`), and
+[CVE-2023-39322](https://github.com/advisories/GHSA-892h-r6cr-53g4) (`crypto/tls`) [[GH-18742](https://github.com/hashicorp/consul/issues/18742)]
+
+IMPROVEMENTS:
+
+* Adds flag -append-filename (which works on values version, dc, node and status) to consul snapshot save command.
+Adding the flag -append-filename version,dc,node,status will add consul version, consul datacenter, node name and leader/follower
+(status) in the file name given in the snapshot save command before the file extension. [[GH-18625](https://github.com/hashicorp/consul/issues/18625)]
+* Reduce the frequency of metric exports from Consul to HCP from every 10s to every 1m [[GH-18584](https://github.com/hashicorp/consul/issues/18584)]
+* api: Add support for listing ACL tokens by service name. [[GH-18667](https://github.com/hashicorp/consul/issues/18667)]
+* checks: It is now possible to configure agent TCP checks to use TLS with
+optional server SNI and mutual authentication. To use TLS with a TCP check, the
+check must enable the `tcp_use_tls` boolean. By default the agent will use the
+TLS configuration in the `tls.default` stanza. [[GH-18381](https://github.com/hashicorp/consul/issues/18381)]
+* command: Adds -since flag in consul debug command which internally calls hcdiag for debug information in the past. [[GH-18797](https://github.com/hashicorp/consul/issues/18797)]
+* log: Currently consul logs files like this consul-{timestamp}.log. This change makes sure that there is always
+consul.log file with the latest logs in it. [[GH-18617](https://github.com/hashicorp/consul/issues/18617)]
+
+BUG FIXES:
+
+* Inherit locality from services when registering sidecar proxies. [[GH-18437](https://github.com/hashicorp/consul/issues/18437)]
+* UI : Nodes list view was breaking for synthetic-nodes. Fix handles non existence of consul-version meta for node. [[GH-18464](https://github.com/hashicorp/consul/issues/18464)]
+* api: Fix `/v1/agent/self` not returning latest configuration [[GH-18681](https://github.com/hashicorp/consul/issues/18681)]
+* ca: Vault provider now cleans up the previous Vault issuer and key when generating a new leaf signing certificate [[GH-18779](https://github.com/hashicorp/consul/issues/18779)] [[GH-18773](https://github.com/hashicorp/consul/issues/18773)]
+* check: prevent go routine leakage when existing Defercheck of same check id is not nil [[GH-18558](https://github.com/hashicorp/consul/issues/18558)]
+* connect: Fix issue where Envoy endpoints would not populate correctly after a snapshot restore. [[GH-18636](https://github.com/hashicorp/consul/issues/18636)]
+* gateways: Fix a bug where gateway to service mappings weren't being cleaned up properly when externally registered proxies were being deregistered. [[GH-18831](https://github.com/hashicorp/consul/issues/18831)]
+* telemetry: emit consul version metric on a regular interval. [[GH-18724](https://github.com/hashicorp/consul/issues/18724)]
+
+## 1.15.6 (September 19, 2023)
+
+SECURITY:
+
+* Upgrade to use Go 1.20.8. This resolves CVEs
+[CVE-2023-39320](https://github.com/advisories/GHSA-rxv8-v965-v333) (`cmd/go`),
+[CVE-2023-39318](https://github.com/advisories/GHSA-vq7j-gx56-rxjh) (`html/template`),
+[CVE-2023-39319](https://github.com/advisories/GHSA-vv9m-32rr-3g55) (`html/template`),
+[CVE-2023-39321](https://github.com/advisories/GHSA-9v7r-x7cv-v437) (`crypto/tls`), and
+[CVE-2023-39322](https://github.com/advisories/GHSA-892h-r6cr-53g4) (`crypto/tls`) [[GH-18742](https://github.com/hashicorp/consul/issues/18742)]
+
+IMPROVEMENTS:
+
+* Adds flag -append-filename (which works on values version, dc, node and status) to consul snapshot save command.
+Adding the flag -append-filename version,dc,node,status will add consul version, consul datacenter, node name and leader/follower
+(status) in the file name given in the snapshot save command before the file extension. [[GH-18625](https://github.com/hashicorp/consul/issues/18625)]
+* Reduce the frequency of metric exports from Consul to HCP from every 10s to every 1m [[GH-18584](https://github.com/hashicorp/consul/issues/18584)]
+* api: Add support for listing ACL tokens by service name. [[GH-18667](https://github.com/hashicorp/consul/issues/18667)]
+* command: Adds -since flag in consul debug command which internally calls hcdiag for debug information in the past. [[GH-18797](https://github.com/hashicorp/consul/issues/18797)]
+* log: Currently consul logs files like this consul-{timestamp}.log. This change makes sure that there is always
+consul.log file with the latest logs in it. [[GH-18617](https://github.com/hashicorp/consul/issues/18617)]
+
+BUG FIXES:
+
+* api: Fix `/v1/agent/self` not returning latest configuration [[GH-18681](https://github.com/hashicorp/consul/issues/18681)]
+* ca: Vault provider now cleans up the previous Vault issuer and key when generating a new leaf signing certificate [[GH-18779](https://github.com/hashicorp/consul/issues/18779)] [[GH-18773](https://github.com/hashicorp/consul/issues/18773)]
+* check: prevent go routine leakage when existing Defercheck of same check id is not nil [[GH-18558](https://github.com/hashicorp/consul/issues/18558)]
+* gateways: Fix a bug where gateway to service mappings weren't being cleaned up properly when externally registered proxies were being deregistered. [[GH-18831](https://github.com/hashicorp/consul/issues/18831)]
+* telemetry: emit consul version metric on a regular interval. [[GH-18724](https://github.com/hashicorp/consul/issues/18724)]
+
+## 1.14.10 (September 19, 2023)
+
+SECURITY:
+
+* Upgrade to use Go 1.20.8. This resolves CVEs
+[CVE-2023-39320](https://github.com/advisories/GHSA-rxv8-v965-v333) (`cmd/go`),
+[CVE-2023-39318](https://github.com/advisories/GHSA-vq7j-gx56-rxjh) (`html/template`),
+[CVE-2023-39319](https://github.com/advisories/GHSA-vv9m-32rr-3g55) (`html/template`),
+[CVE-2023-39321](https://github.com/advisories/GHSA-9v7r-x7cv-v437) (`crypto/tls`), and
+[CVE-2023-39322](https://github.com/advisories/GHSA-892h-r6cr-53g4) (`crypto/tls`) [[GH-18742](https://github.com/hashicorp/consul/issues/18742)]
+
+IMPROVEMENTS:
+
+* Adds flag -append-filename (which works on values version, dc, node and status) to consul snapshot save command.
+Adding the flag -append-filename version,dc,node,status will add consul version, consul datacenter, node name and leader/follower
+(status) in the file name given in the snapshot save command before the file extension. [[GH-18625](https://github.com/hashicorp/consul/issues/18625)]
+* api: Add support for listing ACL tokens by service name. [[GH-18667](https://github.com/hashicorp/consul/issues/18667)]
+* command: Adds -since flag in consul debug command which internally calls hcdiag for debug information in the past. [[GH-18797](https://github.com/hashicorp/consul/issues/18797)]
+* log: Currently consul logs files like this consul-{timestamp}.log. This change makes sure that there is always
+consul.log file with the latest logs in it. [[GH-18617](https://github.com/hashicorp/consul/issues/18617)]
+
+BUG FIXES:
+
+* api: Fix `/v1/agent/self` not returning latest configuration [[GH-18681](https://github.com/hashicorp/consul/issues/18681)]
+* ca: Vault provider now cleans up the previous Vault issuer and key when generating a new leaf signing certificate [[GH-18779](https://github.com/hashicorp/consul/issues/18779)] [[GH-18773](https://github.com/hashicorp/consul/issues/18773)]
+* gateways: Fix a bug where gateway to service mappings weren't being cleaned up properly when externally registered proxies were being deregistered. [[GH-18831](https://github.com/hashicorp/consul/issues/18831)]
+* telemetry: emit consul version metric on a regular interval. [[GH-18724](https://github.com/hashicorp/consul/issues/18724)]
+
 ## 1.16.1 (August 8, 2023)
 
 KNOWN ISSUES:

agent/acl_endpoint_test.go

@@ -1374,7 +1374,7 @@ func TestACL_HTTP(t *testing.T) {
 
 			var list map[string]api.ACLTemplatedPolicyResponse
 			require.NoError(t, json.NewDecoder(resp.Body).Decode(&list))
-			require.Len(t, list, 3)
+			require.Len(t, list, 4)
 
 			require.Equal(t, api.ACLTemplatedPolicyResponse{
 				TemplateName: api.ACLTemplatedPolicyServiceName,

agent/agent_endpoint.go

@@ -1531,6 +1531,9 @@ func (s *HTTPHandlers) AgentToken(resp http.ResponseWriter, req *http.Request) (
 		case "config_file_service_registration":
 			s.agent.tokens.UpdateConfigFileRegistrationToken(args.Token, token_store.TokenSourceAPI)
 
+		case "dns_token", "dns":
+			s.agent.tokens.UpdateDNSToken(args.Token, token_store.TokenSourceAPI)
+
 		default:
 			return HTTPError{StatusCode: http.StatusNotFound, Reason: fmt.Sprintf("Token %q is unknown", target)}
 		}

agent/config/builder.go

@@ -882,6 +882,7 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 			ACLAgentRecoveryToken:          stringVal(c.ACL.Tokens.AgentRecovery),
 			ACLReplicationToken:            stringVal(c.ACL.Tokens.Replication),
 			ACLConfigFileRegistrationToken: stringVal(c.ACL.Tokens.ConfigFileRegistration),
+			ACLDNSToken:                    stringVal(c.ACL.Tokens.DNS),
 		},
 
 		// Autopilot

agent/config/config.go

@@ -778,6 +778,7 @@ type Tokens struct {
 	Default                *string `mapstructure:"default"`
 	Agent                  *string `mapstructure:"agent"`
 	ConfigFileRegistration *string `mapstructure:"config_file_service_registration"`
+	DNS                    *string `mapstructure:"dns"`
 
 	// Enterprise Only
 	ManagedServiceProvider []ServiceProviderToken `mapstructure:"managed_service_provider"`

agent/config/testdata/TestRuntimeConfig_Sanitize.golden

@@ -17,6 +17,7 @@
         "ACLAgentRecoveryToken": "hidden",
         "ACLAgentToken": "hidden",
         "ACLConfigFileRegistrationToken": "hidden",
+        "ACLDNSToken": "hidden",
         "ACLDefaultToken": "hidden",
         "ACLReplicationToken": "hidden",
         "DataDir": "",

agent/consul/type_registry.go

@@ -9,6 +9,7 @@ import (
 	"github.com/hashicorp/consul/internal/mesh"
 	"github.com/hashicorp/consul/internal/resource"
 	"github.com/hashicorp/consul/internal/resource/demo"
+	"github.com/hashicorp/consul/internal/tenancy"
 )
 
 // NewTypeRegistry returns a registry populated with all supported resource
@@ -25,6 +26,7 @@ func NewTypeRegistry() resource.Registry {
 	mesh.RegisterTypes(registry)
 	catalog.RegisterTypes(registry)
 	auth.RegisterTypes(registry)
+	tenancy.RegisterTypes(registry)
 
 	return registry
 }

agent/dns.go

@@ -416,7 +416,7 @@ func (d *DNSServer) handlePtr(resp dns.ResponseWriter, req *dns.Msg) {
 	args := structs.DCSpecificRequest{
 		Datacenter: datacenter,
 		QueryOptions: structs.QueryOptions{
-			Token:      d.agent.tokens.UserToken(),
+			Token:      d.coalesceDNSToken(),
 			AllowStale: cfg.AllowStale,
 		},
 	}
@@ -452,7 +452,7 @@ func (d *DNSServer) handlePtr(resp dns.ResponseWriter, req *dns.Msg) {
 		sargs := structs.ServiceSpecificRequest{
 			Datacenter: datacenter,
 			QueryOptions: structs.QueryOptions{
-				Token:      d.agent.tokens.UserToken(),
+				Token:      d.coalesceDNSToken(),
 				AllowStale: cfg.AllowStale,
 			},
 			ServiceAddress: serviceAddress,
@@ -513,7 +513,7 @@ func (d *DNSServer) handleQuery(resp dns.ResponseWriter, req *dns.Msg) {
 
 	cfg := d.config.Load().(*dnsConfig)
 
-	// Setup the message response
+	// Set up the message response
 	m := new(dns.Msg)
 	m.SetReply(req)
 	m.Compress = !cfg.DisableCompression
@@ -875,7 +875,7 @@ func (d *DNSServer) dispatch(remoteAddr net.Addr, req, resp *dns.Msg, maxRecursi
 			ServiceName:    queryParts[len(queryParts)-1],
 			EnterpriseMeta: locality.EnterpriseMeta,
 			QueryOptions: structs.QueryOptions{
-				Token: d.agent.tokens.UserToken(),
+				Token: d.coalesceDNSToken(),
 			},
 		}
 		if args.PeerName == "" {
@@ -1093,7 +1093,7 @@ func (d *DNSServer) nodeLookup(cfg *dnsConfig, lookup nodeLookup, req, resp *dns
 		PeerName:   lookup.PeerName,
 		Node:       lookup.Node,
 		QueryOptions: structs.QueryOptions{
-			Token:      d.agent.tokens.UserToken(),
+			Token:      d.coalesceDNSToken(),
 			AllowStale: cfg.AllowStale,
 		},
 		EnterpriseMeta: lookup.EnterpriseMeta,
@@ -1425,7 +1425,7 @@ func (d *DNSServer) lookupServiceNodes(cfg *dnsConfig, lookup serviceLookup) (st
 		ServiceTags: serviceTags,
 		TagFilter:   lookup.Tag != "",
 		QueryOptions: structs.QueryOptions{
-			Token:            d.agent.tokens.UserToken(),
+			Token:            d.coalesceDNSToken(),
 			AllowStale:       cfg.AllowStale,
 			MaxAge:           cfg.CacheMaxAge,
 			UseCache:         cfg.UseCache,
@@ -1503,7 +1503,7 @@ func (d *DNSServer) preparedQueryLookup(cfg *dnsConfig, datacenter, query string
 		Datacenter:    datacenter,
 		QueryIDOrName: query,
 		QueryOptions: structs.QueryOptions{
-			Token:      d.agent.tokens.UserToken(),
+			Token:      d.coalesceDNSToken(),
 			AllowStale: cfg.AllowStale,
 			MaxAge:     cfg.CacheMaxAge,
 		},
@@ -2172,3 +2172,11 @@ func (d *DNSServer) resolveCNAME(cfg *dnsConfig, name string, maxRecursionLevel
 	d.logger.Error("all resolvers failed for name", "name", name)
 	return nil
 }
+
+func (d *DNSServer) coalesceDNSToken() string {
+	if d.agent.tokens.DNSToken() != "" {
+		return d.agent.tokens.DNSToken()
+	} else {
+		return d.agent.tokens.UserToken()
+	}
+}

agent/dns_ce_test.go

@@ -10,11 +10,12 @@ import (
 	"context"
 	"testing"
 
+	"github.com/miekg/dns"
+	"github.com/stretchr/testify/require"
+
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/agent/structs"
 	"github.com/hashicorp/consul/testrpc"
-	"github.com/miekg/dns"
-	"github.com/stretchr/testify/require"
 )
 
 func TestDNS_CE_PeeredServices(t *testing.T) {