Merge pull request #10 from mezmo/darinspivey/sync_3.14.1

Merge recent changes from private vector repo
mezmo · Apr 4, 2024 · d4585e2 · d4585e2
2 parents ea0c664 + eeae52e
commit d4585e2
Show file tree

Hide file tree

Showing 10 changed files with 907 additions and 348 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -661,6 +661,7 @@ transforms-metrics-mezmo = [
   "transforms-mezmo_aggregate",
   "transforms-mezmo_tag_cardinality_limit",
   "transforms-mezmo_log_to_metric",
+  "transforms-mezmo_throttle",
   "transforms-protobuf_to_metric",
 ]
 
@@ -680,6 +681,7 @@ transforms-mezmo_log_to_metric = []
 transforms-mezmo_log_clustering = ["dep:lru", "dep:blake2", "dep:base64", "dep:tokio-postgres"]
 transforms-mezmo_log_classification = ["dep:grok"]
 transforms-mezmo_tag_cardinality_limit = ["dep:bloomy", "dep:hashbrown"]
+transforms-mezmo_throttle = []
 transforms-remap = []
 transforms-route = []
 transforms-sample = []

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -6,6 +6,8 @@ def PROJECT_NAME = "vector"
 def CURRENT_BRANCH = currentBranch()
 def DOCKER_REPO = "docker.io/mezmohq"
 
+def BRANCH_BUILD = slugify("${CURRENT_BRANCH}-${BUILD_NUMBER}")
+
 def CREDS = [
   string(
     credentialsId: 'github-api-token',
@@ -48,22 +50,7 @@ pipeline {
     ENVIRONMENT_AUTOBUILD = 'false'
     ENVIRONMENT_TTY = 'false'
     CI = 'true'
-  }
-  post {
-    always {
-      script {
-        if (env.SANITY_BUILD == 'true') {
-          notifySlack(
-            currentBuild.currentResult,
-            [
-              channel: '#pipeline-bots',
-              tokenCredentialId: 'qa-slack-token'
-            ],
-            "`${PROJECT_NAME}` sanity build took ${currentBuild.durationString.replaceFirst(' and counting', '')}."
-          )
-        }
-      }
-    }
+    VECTOR_TARGET = "${BRANCH_BUILD}"
   }
   stages {
     stage('Validate PR Author') {
@@ -78,8 +65,6 @@ pipeline {
       }
     }
 
-
-
     stage('Lint and test release'){
       tools {
         nodejs 'NodeJS 20'
@@ -101,32 +86,36 @@ pipeline {
       }
     }
 
-    stage('vdev Check'){
-      when {
-        changeRequest() // Only do this during PRs. It's about a 15-min wait.
-      }
+    stage('Unit test'){
+      // Important: do one step serially since it'll be the one to prepare the testing container
+      // and install the rust toolchain in it. Volume mounts are created here, too.
       steps {
         sh """
-          make check ENVIRONMENT=true
-          make check-fmt ENVIRONMENT=true
+          make test ENVIRONMENT=true
         """
       }
     }
 
-    stage('Code'){
+    stage('Checks'){
+      // All `make ENVIRONMENT=true` steps should now use the existing container
       parallel {
-        stage('Lint'){
+        stage('check-clippy'){
           steps {
             sh """
               make check-clippy ENVIRONMENT=true
               make check-scripts ENVIRONMENT=true
             """
           }
         }
-        stage('Check Deny'){
-          when {
-            changeRequest() // PRs only to speed up dev flows. These can be fixed then if they're actionable.
+        stage('check-fmt'){
+          steps {
+            sh """
+              make check ENVIRONMENT=true
+              make check-fmt ENVIRONMENT=true
+            """
           }
+        }
+        stage('check-deny'){
           steps {
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
               sh """
@@ -135,14 +124,7 @@ pipeline {
             }
           }
         }
-        stage('Unit test'){
-          steps {
-            sh """
-              make test ENVIRONMENT=true
-            """
-          }
-        }
-        stage('Test build container image') {
+        stage('image test') {
           when {
             changeRequest() // Only do this during PRs because it can take 30 minutes
           }
@@ -151,7 +133,7 @@ pipeline {
               buildx.build(
                 project: PROJECT_NAME
               , push: false
-              , tags: [slugify("${CURRENT_BRANCH}-${BUILD_NUMBER}")]
+              , tags: [BRANCH_BUILD]
               , dockerfile: "distribution/docker/mezmo/Dockerfile"
               , docker_repo: DOCKER_REPO
               )
@@ -213,4 +195,24 @@ pipeline {
       }
     }
   }
+  post {
+    always {
+      // Clear disk space by removing the `target` volume mount where the binaries are stored.
+      // The volume is unique to the current build, so there should be no "in use" errors.
+      sh 'make target-clean'
+
+      script {
+        if (env.SANITY_BUILD == 'true') {
+          notifySlack(
+            currentBuild.currentResult,
+            [
+              channel: '#pipeline-bots',
+              tokenCredentialId: 'qa-slack-token'
+            ],
+            "`${PROJECT_NAME}` sanity build took ${currentBuild.durationString.replaceFirst(' and counting', '')}."
+          )
+        }
+      }
+    }
+  }
 }
diff --git a/MEZMO_CHANGELOG.md b/MEZMO_CHANGELOG.md
@@ -1,4 +1,5 @@
 ## [3.13.6](https://github.com/mezmo/vector/compare/v3.13.5...v3.13.6) (2024-04-03)
+## [3.14.1](https://github.com/answerbook/vector/compare/v3.14.0...v3.14.1) (2024-04-03)
 
 
 ### Bug Fixes
@@ -18,11 +19,48 @@
 ### Chores
 
 * **ci**: Publish to `docker.io` and use public repos [631a8ab](https://github.com/mezmo/vector/commit/631a8ab6b3f9482610b80c74ac9d290e5c63787d) - Darin Spivey [LOG-18250](https://logdna.atlassian.net/browse/LOG-18250)
+* **ci**: Clean disk space after ci runs [ci skip] [7963582](https://github.com/answerbook/vector/commit/796358215aaa5eecd597addbd39a6e5fad998a13) - Darin Spivey [LOG-19590](https://logdna.atlassian.net/browse/LOG-19590)
 
 
 ### Miscellaneous
 
 * Merge pull request #7 from mezmo/darinspivey/LOG-18250 [7f3158e](https://github.com/mezmo/vector/commit/7f3158e172e6eae32ba00f608f33718db4316c1c) - GitHub [LOG-18250](https://logdna.atlassian.net/browse/LOG-18250)
+* Merge pull request #439 from answerbook/darinspivey/LOG-19590 [b658874](https://github.com/answerbook/vector/commit/b658874d925621326410d26a187ef3087efa24a1) - GitHub [LOG-19590](https://logdna.atlassian.net/browse/LOG-19590)
+
+# [3.14.0](https://github.com/answerbook/vector/compare/v3.13.3...v3.14.0) (2024-04-02)
+
+
+### Bug Fixes
+
+* **throttle**: use ms for window config [f898c1f](https://github.com/answerbook/vector/commit/f898c1ff8b873a63937e061d722a9499cd8da9b4) - Mike Del Tito [LOG-19565](https://logdna.atlassian.net/browse/LOG-19565)
+
+
+### Features
+
+* **throttle**: add state persistence [ad882f2](https://github.com/answerbook/vector/commit/ad882f26c0f88b660b75b6be9f9c443fac554620) - Mike Del Tito [LOG-19565](https://logdna.atlassian.net/browse/LOG-19565)
+* **throttle**: reimplement throttle transform with basic rate limiting [066cac3](https://github.com/answerbook/vector/commit/066cac3957882804c6174f11924e1b2ea4a0b91f) - Mike Del Tito [LOG-19565](https://logdna.atlassian.net/browse/LOG-19565)
+
+
+### Miscellaneous
+
+* Merge pull request #438 from answerbook/mdeltito/LOG-19565 [f8854d6](https://github.com/answerbook/vector/commit/f8854d6c07b4cc173531acb70ced6a2aea424b17) - GitHub [LOG-19565](https://logdna.atlassian.net/browse/LOG-19565)
+
+## [3.13.3](https://github.com/answerbook/vector/compare/v3.13.2...v3.13.3) (2024-03-27)
+
+
+### Chores
+
+* **build**: Use open source vrl fork [347e20f](https://github.com/answerbook/vector/commit/347e20f28b0480c707868f3cc8713b7e7d534ade) - Dan Hable [LOG-19155](https://logdna.atlassian.net/browse/LOG-19155)
+
+
+### Miscellaneous
+
+* Merge pull request #6 from mezmo/darinspivey/answerbook_sync [bd66904](https://github.com/answerbook/vector/commit/bd66904d93efc261909215b5c3141cf21540fcf4) - GitHub
+* Merge remote-tracking branch 'answerbook/master' into darinspivey/answerbook_sync [98280d6](https://github.com/answerbook/vector/commit/98280d69bfc94fd4e1261841e7eeaca13df1a2e2) - Darin Spivey
+* Merge pull request #5 from mezmo/holmberg/LOG-19506 [35fed6f](https://github.com/answerbook/vector/commit/35fed6fd25638e558f15404fa75b1a818caa6247) - GitHub [LOG-19506](https://logdna.atlassian.net/browse/LOG-19506)
+* Merge pull request #3 from mezmo/edge1 [a049b0c](https://github.com/answerbook/vector/commit/a049b0c62ba1705b958fb56d45abc3c1b970d0a3) - GitHub
+* Merge branch 'master' into edge1 [596ba22](https://github.com/answerbook/vector/commit/596ba22be5bf186e88ea8e1798d1b6587dec8ebe) - GitHub
+* Merge pull request #2 from mezmo/dhable/LOG-19155 [7ddc8c2](https://github.com/answerbook/vector/commit/7ddc8c2f5fb8c29a10610522d533b36ac27e7087) - GitHub [LOG-19155](https://logdna.atlassian.net/browse/LOG-19155)
 
 ## [3.13.2](https://github.com/answerbook/vector/compare/v3.13.1...v3.13.2) (2024-03-26)
 

diff --git a/Makefile b/Makefile
@@ -94,6 +94,8 @@ export VERSION ?= $(shell command -v cargo >/dev/null && cargo vdev version || e
 # Set if you are on the CI and actually want the things to happen. (Non-CI users should never set this.)
 export CI ?= false
 
+export VECTOR_TARGET ?= vector-target
+
 export RUST_VERSION ?= $(shell grep channel rust-toolchain.toml | cut -d '"' -f 2)
 
 FORMATTING_BEGIN_YELLOW = \033[0;33m
@@ -162,7 +164,9 @@ define ENVIRONMENT_EXEC
 			$(if $(ENVIRONMENT_NETWORK),--network $(ENVIRONMENT_NETWORK),) \
 			--mount type=bind,source=${CURRENT_DIR},target=/git/vectordotdev/vector \
 			$(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=/var/run/docker.sock$(COMMA)target=/var/run/docker.sock,) \
-			--mount type=volume,source=vector-target,target=/git/vectordotdev/vector/target \
+			--mount type=volume,source=${VECTOR_TARGET},target=/git/vectordotdev/vector/target \
+			--mount type=volume,source=vector-cargo-cache,target=/root/.cargo \
+			--mount type=volume,source=vector-rustup-cache,target=/root/.rustup \
 			$(foreach publish,$(ENVIRONMENT_PUBLISH),--publish $(publish)) \
 			$(ENVIRONMENT_UPSTREAM)
 endef
@@ -204,9 +208,14 @@ environment:
 environment-prepare: ## Prepare the Vector dev shell using $CONTAINER_TOOL.
 	${ENVIRONMENT_PREPARE}
 
+.PHONY: target-clean
+target-clean: ## Clean just the target volume, and leave toolchain/cargo in tact
+	@echo "Removing vector target volume: ${VECTOR_TARGET}"
+	@$(CONTAINER_TOOL) volume rm -f ${VECTOR_TARGET}
+
 .PHONY: environment-clean
 environment-clean: ## Clean the Vector dev shell using $CONTAINER_TOOL.
-	@$(CONTAINER_TOOL) volume rm -f vector-target vector-cargo-cache vector-rustup-cache
+	@$(CONTAINER_TOOL) volume rm -f ${VECTOR_TARGET} vector-cargo-cache vector-rustup-cache
 	@$(CONTAINER_TOOL) rmi $(ENVIRONMENT_UPSTREAM) || true
 
 .PHONY: environment-push

diff --git a/commitlint.config.js b/commitlint.config.js
@@ -0,0 +1,7 @@
+'use strict'
+
+module.exports = {
+  rules: {
+   'body-max-line-length': [2, 'always', 80]
+  }
+}