From 6e3e3d57f9c67e4a4683cba22cf838e0f33588d5 Mon Sep 17 00:00:00 2001 From: Mike Del Tito Date: Fri, 5 Apr 2024 14:33:40 -0400 Subject: [PATCH 1/4] chore: reenable `profiling` feature for jemalloc This was inadvertently removed as part of an upgrade. Re-enable this to allow heap dumps to be taken. Ref: LOG-19647 --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 4af01417c..918e8af62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -318,7 +318,7 @@ stream-cancel = { version = "0.8.1", default-features = false } strip-ansi-escapes = { version = "0.2.0", default-features = false } syslog = { version = "6.1.0", default-features = false, optional = true } urlencoding = { version = "2.1.0", default-features = false } -tikv-jemallocator = { version = "0.5.4", default-features = false, optional = true } +tikv-jemallocator = { version = "0.5.4", default-features = false, optional = true, features = ["profiling"] } tokio-postgres = { version = "0.7.10", default-features = false, features = ["runtime", "with-chrono-0_4"], optional = true } tokio-tungstenite = {version = "0.20.1", default-features = false, features = ["connect"], optional = true} toml = { version = "0.8.6", default-features = false, features = ["parse", "display"] } From c8a2366b81696883eb163971a13ba1d94b3d28e8 Mon Sep 17 00:00:00 2001 From: LogDNA Bot Date: Fri, 5 Apr 2024 23:20:38 +0000 Subject: [PATCH 2/4] chore(release): 3.14.2 [skip ci] ## [3.14.2](https://github.com/answerbook/vector/compare/v3.14.1...v3.14.2) (2024-04-05) ### Chores * reenable `profiling` feature for jemalloc [6e3e3d5](https://github.com/answerbook/vector/commit/6e3e3d57f9c67e4a4683cba22cf838e0f33588d5) - Mike Del Tito [LOG-19647](https://logdna.atlassian.net/browse/LOG-19647) ### Miscellaneous * Merge pull request #442 from answerbook/mdeltito/LOG-19647 [10e26f6](https://github.com/answerbook/vector/commit/10e26f6e4cb0d7fd311f4c3ab0e432f208903414) - GitHub [LOG-19647](https://logdna.atlassian.net/browse/LOG-19647) --- MEZMO_CHANGELOG.md | 12 ++++++++++++ package.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/MEZMO_CHANGELOG.md b/MEZMO_CHANGELOG.md index af1d63f3f..59868559b 100644 --- a/MEZMO_CHANGELOG.md +++ b/MEZMO_CHANGELOG.md @@ -1,3 +1,15 @@ +## [3.14.2](https://github.com/answerbook/vector/compare/v3.14.1...v3.14.2) (2024-04-05) + + +### Chores + +* reenable `profiling` feature for jemalloc [6e3e3d5](https://github.com/answerbook/vector/commit/6e3e3d57f9c67e4a4683cba22cf838e0f33588d5) - Mike Del Tito [LOG-19647](https://logdna.atlassian.net/browse/LOG-19647) + + +### Miscellaneous + +* Merge pull request #442 from answerbook/mdeltito/LOG-19647 [10e26f6](https://github.com/answerbook/vector/commit/10e26f6e4cb0d7fd311f4c3ab0e432f208903414) - GitHub [LOG-19647](https://logdna.atlassian.net/browse/LOG-19647) + ## [3.14.1](https://github.com/answerbook/vector/compare/v3.14.0...v3.14.1) (2024-04-03) diff --git a/package.json b/package.json index c051f52ff..dc6c9e92d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "vector", - "version": "3.14.1", + "version": "3.14.2", "description": "Vector is a high-performance, end-to-end (agent & aggregator) observability data pipeline", "repository": { "type": "git", From ee0749c296fdeef2220b889b6bce8ec74771f040 Mon Sep 17 00:00:00 2001 From: Darin Spivey Date: Fri, 5 Apr 2024 11:04:43 -0400 Subject: [PATCH 3/4] fix(ci): Make all vector dev volumnes unique For running multiple jobs in CI, it will help to have all volumes be owned by whatever branch is running. Similar to `VECTOR_TARGET`, add a dynamic volume name for rust and cargo caches. Clean up those volumes upon CI exit by calling `environment-clean` which will also remove any stopped/unused containers. Ref: LOG-19643 --- Jenkinsfile | 60 ++++++++++++++++++++++++++++------------------------- Makefile | 22 +++++++++++++------- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fd46f3197..f813b3d88 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -55,27 +55,19 @@ pipeline { ENVIRONMENT_AUTOBUILD = 'false' ENVIRONMENT_TTY = 'false' CI = 'true' - VECTOR_TARGET = "${BRANCH_BUILD}" + VECTOR_TARGET = "${BRANCH_BUILD}-target" + VECTOR_CARGO_CACHE = "${BRANCH_BUILD}-cargo" + VECTOR_RUSTUP_CACHE = "${BRANCH_BUILD}-rustup" } stages { - stage('Setup') { + stage('Release Tool') { steps { sh 'make release-tool' } } - stage('Check'){ - // Important: do this step serially since it'll be the one to prepare the testing container - // and install the rust toolchain in it. Volume mounts are created here, too. - steps { - sh """ - make check ENVIRONMENT=true - make check-fmt ENVIRONMENT=true - """ - } - } stage('Lint and test release'){ tools { - nodejs 'NodeJS 16' + nodejs 'NodeJS 20' } environment { GIT_BRANCH = "${CURRENT_BRANCH}" @@ -94,10 +86,19 @@ pipeline { sh './release-tool test' } } - stage('Lint and Test') { + stage('Unit test'){ + // Important: do one step serially since it'll be the one to prepare the testing container + // and install the rust toolchain in it. Volume mounts are created here, too. + steps { + sh """ + make test ENVIRONMENT=true + """ + } + } + stage('Checks') { // All `make ENVIRONMENT=true` steps should now use the existing container parallel { - stage('Lint'){ + stage('check-clippy'){ steps { sh """ make check-clippy ENVIRONMENT=true @@ -105,7 +106,17 @@ pipeline { """ } } - stage('Deny'){ + stage('check-fmt'){ + // Important: do this step serially since it'll be the one to prepare the testing container + // and install the rust toolchain in it. Volume mounts are created here, too. + steps { + sh """ + make check ENVIRONMENT=true + make check-fmt ENVIRONMENT=true + """ + } + } + stage('check-deny'){ steps { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh """ @@ -114,14 +125,7 @@ pipeline { } } } - stage('Unit test'){ - steps { - sh """ - make test ENVIRONMENT=true - """ - } - } - stage('Test build container image') { + stage('image test') { when { changeRequest() // Only do this during PRs because it can take 30 minutes } @@ -167,7 +171,7 @@ pipeline { } } tools { - nodejs 'NodeJS 16' + nodejs 'NodeJS 20' } steps { script { @@ -196,9 +200,9 @@ pipeline { } post { always { - // Clear disk space by removing the `target` volume mount where the binaries are stored. - // The volume is unique to the current build, so there should be no "in use" errors. - sh 'make target-clean' + // Clear disk space by removing the test container and all of its volumes. + // The container and volumes are unique to the current build, so there should be no "in use" errors. + sh 'make environment-clean' script { if (env.SANITY_BUILD == 'true') { diff --git a/Makefile b/Makefile index c2b3710fa..6571d28cb 100644 --- a/Makefile +++ b/Makefile @@ -95,6 +95,8 @@ export VERSION ?= $(shell command -v cargo >/dev/null && cargo vdev version || e export CI ?= false export VECTOR_TARGET ?= vector-target +export VECTOR_CARGO_CACHE ?= vector-cargo-cache +export VECTOR_RUSTUP_CACHE ?= vector-rustup-cache export RUST_VERSION ?= $(shell grep channel rust-toolchain.toml | cut -d '"' -f 2) @@ -149,6 +151,7 @@ BUILD_TAG ?= CONTAINER_ID = vector-environment-$(shell echo $(BUILD_TAG) | sed -E 's/[^a-zA-Z0-9_.-]/-/g')$(shell date +%s)-$(shell echo $$PPID) # We use a volume here as non-Linux hosts are extremely slow to share disks, and Linux hosts tend to get permissions clobbered. +# Note that `--rm` does not remove named volumes, so these will provide caching until removed with `evironment-clean` define ENVIRONMENT_EXEC ${ENVIRONMENT_PREPARE} @echo "Entering environment..." @@ -167,8 +170,8 @@ define ENVIRONMENT_EXEC --mount type=bind,source=${CURRENT_DIR}/scripts/environment/entrypoint.sh,target=/entrypoint.sh \ $(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=/var/run/docker.sock$(COMMA)target=/var/run/docker.sock,) \ --mount type=volume,source=${VECTOR_TARGET},target=/git/vectordotdev/vector/target \ - --mount type=volume,source=vector-cargo-cache,target=/root/.cargo \ - --mount type=volume,source=vector-rustup-cache,target=/root/.rustup \ + --mount type=volume,source=${VECTOR_CARGO_CACHE},target=/root/.cargo \ + --mount type=volume,source=${VECTOR_RUSTUP_CACHE},target=/root/.rustup \ $(foreach publish,$(ENVIRONMENT_PUBLISH),--publish $(publish)) \ $(ENVIRONMENT_UPSTREAM) endef @@ -210,14 +213,17 @@ environment: environment-prepare: ## Prepare the Vector dev shell using $CONTAINER_TOOL. ${ENVIRONMENT_PREPARE} -.PHONY: target-clean -target-clean: ## Clean just the target volume, and leave toolchain/cargo in tact - @echo "Removing vector target volume: ${VECTOR_TARGET}" - @$(CONTAINER_TOOL) volume rm -f ${VECTOR_TARGET} - .PHONY: environment-clean environment-clean: ## Clean the Vector dev shell using $CONTAINER_TOOL. - @$(CONTAINER_TOOL) volume rm -f ${VECTOR_TARGET} vector-cargo-cache vector-rustup-cache + @echo "\nList of active volumes" + @$(CONTAINER_TOOL) volume ls + @echo "\nList of all containers" + @$(CONTAINER_TOOL) ps -a + @echo "\nRemoving vector dev volumes" + @$(CONTAINER_TOOL) volume rm -f ${VECTOR_TARGET} ${VECTOR_CARGO_CACHE} ${VECTOR_RUSTUP_CACHE} + @echo "\nRemoving stopped containers" + @$(CONTAINER_TOOL) container prune + @echo "\nRemoving vector dev image (if possible)" @$(CONTAINER_TOOL) rmi $(ENVIRONMENT_UPSTREAM) || true .PHONY: environment-push From 846c1ab63ba7cbf0fc9c491e0d6dfbc2d40f4328 Mon Sep 17 00:00:00 2001 From: LogDNA Bot Date: Mon, 8 Apr 2024 15:17:13 +0000 Subject: [PATCH 4/4] chore(release): 3.14.3 [skip ci] ## [3.14.3](https://github.com/answerbook/vector/compare/v3.14.2...v3.14.3) (2024-04-08) ### Bug Fixes * **ci**: Make all vector dev volumnes unique [ee0749c](https://github.com/answerbook/vector/commit/ee0749c296fdeef2220b889b6bce8ec74771f040) - Darin Spivey [LOG-19643](https://logdna.atlassian.net/browse/LOG-19643) --- MEZMO_CHANGELOG.md | 7 +++++++ package.json | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/MEZMO_CHANGELOG.md b/MEZMO_CHANGELOG.md index 59868559b..f0e6dfa33 100644 --- a/MEZMO_CHANGELOG.md +++ b/MEZMO_CHANGELOG.md @@ -1,3 +1,10 @@ +## [3.14.3](https://github.com/answerbook/vector/compare/v3.14.2...v3.14.3) (2024-04-08) + + +### Bug Fixes + +* **ci**: Make all vector dev volumnes unique [ee0749c](https://github.com/answerbook/vector/commit/ee0749c296fdeef2220b889b6bce8ec74771f040) - Darin Spivey [LOG-19643](https://logdna.atlassian.net/browse/LOG-19643) + ## [3.14.2](https://github.com/answerbook/vector/compare/v3.14.1...v3.14.2) (2024-04-05) diff --git a/package.json b/package.json index dc6c9e92d..aa384a90f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "vector", - "version": "3.14.2", + "version": "3.14.3", "description": "Vector is a high-performance, end-to-end (agent & aggregator) observability data pipeline", "repository": { "type": "git",