Skip to content

Commit

Permalink
Merge branch 'main' into matt/fim-ebpf
Browse files Browse the repository at this point in the history
  • Loading branch information
mmat11 committed Feb 9, 2024
2 parents ba46a56 + 4c75049 commit b9982c8
Show file tree
Hide file tree
Showing 15 changed files with 199 additions and 60 deletions.
2 changes: 1 addition & 1 deletion .buildkite/auditbeat/auditbeat-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -134,4 +134,4 @@ steps:

steps:
- label: Package pipeline
commands: ".buildkite/auditbeat/scripts/package-step.sh | buildkite-agent pipeline upload"
commands: ".buildkite/auditbeat/scripts/package-step.sh"
4 changes: 3 additions & 1 deletion .buildkite/auditbeat/scripts/package-step.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ changeset="^auditbeat/
^\.buildkite/auditbeat/"

if are_files_changed "$changeset"; then
cat <<-YAML
bk_pipeline=$(cat <<-YAML
steps:
- label: ":ubuntu: Packaging Linux X86"
key: "package-linux-x86"
Expand Down Expand Up @@ -43,6 +43,8 @@ if are_files_changed "$changeset"; then
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "t4g.large"
YAML
)
echo "${bk_pipeline}" | buildkite-agent pipeline upload
else
buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
exit 0
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/filebeat/filebeat-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -137,4 +137,4 @@ steps:

steps:
- label: Package pipeline
commands: ".buildkite/filebeat/scripts/package-step.sh | buildkite-agent pipeline upload"
commands: ".buildkite/filebeat/scripts/package-step.sh"
4 changes: 3 additions & 1 deletion .buildkite/filebeat/scripts/package-step.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ changeset="^filebeat/
^\.buildkite/filebeat/"

if are_files_changed "$changeset"; then
cat <<-YAML
bk_pipeline=$(cat <<-YAML
steps:
- label: ":ubuntu: Packaging Linux X86"
key: "package-linux-x86"
Expand Down Expand Up @@ -43,6 +43,8 @@ if are_files_changed "$changeset"; then
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "t4g.large"
YAML
)
echo "${bk_pipeline}" | buildkite-agent pipeline upload
else
buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
exit 0
Expand Down
2 changes: 1 addition & 1 deletion .buildkite/heartbeat/heartbeat-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,4 +139,4 @@ steps:

steps:
- label: Package pipeline
commands: ".buildkite/heartbeat/scripts/package-step.sh | buildkite-agent pipeline upload"
commands: ".buildkite/heartbeat/scripts/package-step.sh"
4 changes: 3 additions & 1 deletion .buildkite/heartbeat/scripts/package-step.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ changeset="^heartbeat/
^\.buildkite/heartbeat/"

if are_files_changed "$changeset"; then
cat <<-YAML
bk_pipeline=$(cat <<-YAML
steps:
- label: ":ubuntu: Packaging Linux X86"
key: "package-linux-x86"
Expand Down Expand Up @@ -43,6 +43,8 @@ if are_files_changed "$changeset"; then
imagePrefix: "${IMAGE_UBUNTU_ARM_64}"
instanceType: "t4g.large"
YAML
)
echo "${bk_pipeline}" | buildkite-agent pipeline upload
else
buildkite-agent annotate "No required files changed. Skipped packaging" --style 'warning' --context 'ctx-warning'
exit 0
Expand Down
3 changes: 1 addition & 2 deletions .buildkite/hooks/pre-command
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,7 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "filebeat" || "$BUILDKITE_PIPELINE_SLUG" ==
fi
fi


if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" || "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then
source .buildkite/scripts/setenv.sh
if [[ "${BUILDKITE_COMMAND}" =~ ^buildkite-agent ]]; then
echo "Skipped pre-command when running the Upload pipeline"
Expand Down
4 changes: 0 additions & 4 deletions .buildkite/libbeat/pipeline.libbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,3 @@ steps:
- label: ":linux: Load dynamic Libbeat pipeline"
key: "libbeat-pipeline"
command: ".buildkite/scripts/generate_libbeat_pipeline.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
5 changes: 0 additions & 5 deletions .buildkite/metricbeat/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ env:
IMAGE_WIN_2019: "family/core-windows-2019"
IMAGE_WIN_2022: "family/core-windows-2022"
IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
GO_AGENT_IMAGE: "golang:${GO_VERSION}"
BEATS_PROJECT_NAME: "metricbeat"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
Expand Down Expand Up @@ -46,7 +45,3 @@ steps:
- label: ":linux: Load dynamic metricbeat pipeline"
key: "metricbeat-pipeline"
command: ".buildkite/scripts/generate_metricbeat_pipeline.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
3 changes: 0 additions & 3 deletions .buildkite/packetbeat/pipeline.packetbeat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,3 @@ steps:
- label: ":linux: Load dynamic packetbeat pipeline"
key: "packetbeat-pipeline"
command: ".buildkite/scripts/generate_packetbeat_pipeline.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
46 changes: 25 additions & 21 deletions .buildkite/scripts/common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,12 @@ platform_type="$(uname)"
platform_type_lowercase=$(echo "$platform_type" | tr '[:upper:]' '[:lower:]')
arch_type="$(uname -m)"
GITHUB_PR_TRIGGER_COMMENT=${GITHUB_PR_TRIGGER_COMMENT:-""}
GITHUB_PR_LABELS=${GITHUB_PR_LABELS:-""}
ONLY_DOCS=${ONLY_DOCS:-"true"}

[ -z "${runLibbeat+x}" ] && runLibbeat="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
[ -z "${runMetricbeat+x}" ] && runMetricbeat="$(buildkite-agent meta-data get runMetricbeat --default ${runMetricbeat:-"false"})"
[ -z "${runPacketbeat+x}" ] && runPacketbeat="$(buildkite-agent meta-data get runPacketbeat --default ${runPacketbeat:-"false"})"
[ -z "${runWinlogbeat+x}" ] && runWinlogbeat="$(buildkite-agent meta-data get runWinlogbeat --default ${runWinlogbeat:-"false"})"
[ -z "${runLibBeatArmTest+x}" ] && runLibBeatArmTest="$(buildkite-agent meta-data get runLibbeat --default ${runLibbeat:-"false"})"
[ -z "${runPacketbeatArmTest+x}" ] && runPacketbeatArmTest="$(buildkite-agent meta-data get runPacketbeatArmTest --default ${runPacketbeatArmTest:-"false"})"
[ -z "${runMetricbeatMacOsTests+x}" ] && runMetricbeatMacOsTests="$(buildkite-agent meta-data get runMetricbeatMacOsTests --default ${runMetricbeatMacOsTests:-"false"})"
Expand All @@ -29,6 +30,10 @@ packetbeat_changeset=(
"^packetbeat/.*"
)

winlogbeat_changeset=(
"^winlogbeat/.*"
)

oss_changeset=(
"^go.mod"
"^pytest.ini"
Expand Down Expand Up @@ -231,18 +236,23 @@ are_changed_only_paths() {

are_conditions_met_mandatory_tests() {
if are_paths_changed "${oss_changeset[@]}" || are_paths_changed "${ci_changeset[@]}" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L3-L12
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
return 0
fi
return 0
fi
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
if are_paths_changed "${metricbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test metricbeat" || "${GITHUB_PR_LABELS}" =~ Metricbeat || "${runMetricbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" ]]; then
if are_paths_changed "${libbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test libbeat" || "${GITHUB_PR_LABELS}" =~ libbeat || "${runLibbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if are_paths_changed "${packetbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test packetbeat" || "${GITHUB_PR_LABELS}" =~ Packetbeat || "${runPacketbeat}" == "true" ]]; then
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-winlogbeat" ]]; then
if are_paths_changed "${winlogbeat_changeset[@]}" || [[ "${GITHUB_PR_TRIGGER_COMMENT}" == "/test winlogbeat" || "${GITHUB_PR_LABELS}" =~ Winlogbeat || "${runWinlogbeat}" == "true" ]]; then
return 0
fi
fi
return 1
Expand Down Expand Up @@ -280,14 +290,8 @@ are_conditions_met_macos_tests() {

are_conditions_met_packaging() {
if are_conditions_met_mandatory_tests; then #from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/Jenkinsfile#L145-L171
if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" ]]; then
if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then # from https://github.com/elastic/beats/blob/c5e79a25d05d5bdfa9da4d187fe89523faa42afc/metricbeat/Jenkinsfile.yml#L101-L103
return 0
fi
elif [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-packetbeat" ]]; then
if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
return 0
fi
if [[ "${BUILDKITE_TAG}" == "" || "${BUILDKITE_PULL_REQUEST}" != "" ]]; then
return 0
fi
fi
return 1
Expand Down
109 changes: 109 additions & 0 deletions .buildkite/scripts/generate_winlogbeat_pipeline.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
#!/usr/bin/env bash

source .buildkite/scripts/common.sh

set -euo pipefail

pipelineName="pipeline.winlogbeat-dynamic.yml"

echo "Add the mandatory and extended tests without additional conditions into the pipeline"
if are_conditions_met_mandatory_tests; then
cat > $pipelineName <<- YAML
steps:
- group: "Mandatory Tests"
key: "mandatory-tests"
steps:
- label: ":negative_squared_cross_mark: Cross compile"
key: "mandatory-cross-compile"
command: ".buildkite/scripts/crosscompile.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_DEFAULT_MACHINE_TYPE}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
- label: ":windows: Windows 2016/2019/2022 Unit Tests - {{matrix.image}}"
command: ".buildkite/scripts/win_unit_tests.ps1"
key: "mandatory-win-unit-tests"
agents:
provider: "gcp"
image: "{{matrix.image}}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
matrix:
setup:
image:
- "${IMAGE_WIN_2016}"
- "${IMAGE_WIN_2019}"
- "${IMAGE_WIN_2022}"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
# echo "Add the extended windows tests into the pipeline"
# TODO: ADD conditions from the main pipeline
- group: "Extended Windows Tests"
key: "extended-win-tests"
steps:
- label: ":windows: Windows 10 Unit Tests"
key: "extended-win-10-unit-tests"
command: ".buildkite/scripts/win_unit_tests.ps1"
agents:
provider: "gcp"
image: "${IMAGE_WIN_10}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
- label: ":windows: Windows 11 Unit Tests"
key: "extended-win-11-unit-tests"
command: ".buildkite/scripts/win_unit_tests.ps1"
agents:
provider: "gcp"
image: "${IMAGE_WIN_11}"
machine_type: "${GCP_WIN_MACHINE_TYPE}"
disk_size: 100
disk_type: "pd-ssd"
artifact_paths: "${BEATS_PROJECT_NAME}/build/*.*"
YAML
else
echo "The conditions don't match to requirements for generating pipeline steps."
exit 0
fi

echo "Check and add the Packaging into the pipeline"
if are_conditions_met_packaging; then
cat >> $pipelineName <<- YAML
- wait: ~
depends_on:
- step: "mandatory-tests"
allow_failure: false
- group: "Packaging" # TODO: check conditions for future the main pipeline migration: https://github.com/elastic/beats/pull/28589
key: "packaging"
steps:
- label: ":linux: Packaging Linux"
key: "packaging-linux"
command: ".buildkite/scripts/packaging.sh"
agents:
provider: "gcp"
image: "${IMAGE_UBUNTU_X86_64}"
machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
env:
PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
YAML
fi

echo "--- Printing dynamic steps" #TODO: remove if the pipeline is public
cat $pipelineName

echo "--- Loading dynamic steps"
buildkite-agent pipeline upload $pipelineName
9 changes: 6 additions & 3 deletions .buildkite/scripts/win_unit_tests.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,12 @@ function withChoco {
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
}
function withGolang($version) {
Write-Host "-- Install golang $version --"
choco install -y golang --version=$version
refreshenv
$downloadPath = Join-Path $env:TEMP "go_installer.msi"
$goInstallerUrl = "https://golang.org/dl/go$version.windows-amd64.msi"
Invoke-WebRequest -Uri $goInstallerUrl -OutFile $downloadPath
Start-Process -FilePath "msiexec.exe" -ArgumentList "/i $downloadPath /quiet" -Wait
$goBinPath = "${env:ProgramFiles}\Go\bin"
$env:Path += ";$goBinPath"
go version
}
function withPython($version) {
Expand Down
34 changes: 32 additions & 2 deletions .buildkite/winlogbeat/pipeline.winlogbeat.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,35 @@
# yaml-language-server: $schema=https://raw.githubusercontent.com/buildkite/pipeline-schema/main/schema.json
env:
IMAGE_UBUNTU_X86_64: "family/core-ubuntu-2204"
IMAGE_WIN_10: "family/general-windows-10"
IMAGE_WIN_11: "family/general-windows-11"
IMAGE_WIN_2016: "family/core-windows-2016"
IMAGE_WIN_2019: "family/core-windows-2019"
IMAGE_WIN_2022: "family/core-windows-2022"
GCP_DEFAULT_MACHINE_TYPE: "c2d-highcpu-8"
GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
GCP_WIN_MACHINE_TYPE: "n2-standard-8"
BEATS_PROJECT_NAME: "winlogbeat"

steps:
- label: "Example test"
command: echo "Hello!"

- input: "Input Parameters"
key: "input-run-all-stages"
fields:
- select: "Packetbeat - runWinlogbeat"
key: "runWinlogbeat"
options:
- label: "True"
value: "true"
- label: "False"
value: "false"
default: "false"
if: "build.source == 'ui'"

- wait: ~
if: "build.source == 'ui'"
allow_dependency_failure: false

- label: ":linux: Load dynamic winlogbeat pipeline"
key: "winlogbeat-pipeline"
command: ".buildkite/scripts/generate_winlogbeat_pipeline.sh"
Loading

0 comments on commit b9982c8

Please sign in to comment.