From 328670be543c281a3d3bc6906f9a11b9ad446c22 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Wed, 12 Jun 2024 12:59:28 +0530 Subject: [PATCH 1/9] feature: tag events that come from a filestream with `take_over: true` (#39828) * filestream: tag events with `take_over: true` * filestream: modify test cases * add comments * update documentation * Update filebeat/input/filestream/input_test.go Co-authored-by: Tiago Queiroz * add changelog --------- Co-authored-by: Tiago Queiroz --- CHANGELOG.next.asciidoc | 1 + .../docs/howto/migrate-to-filestream.asciidoc | 5 +++ filebeat/input/filestream/input.go | 8 ++++ filebeat/input/filestream/input_test.go | 43 +++++++++++++++++++ 4 files changed, 57 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e1214aa0e27..8e5cd3497e1 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -42,6 +42,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Removed deprecated Sophos UTM from Beats. Use the https://docs.elastic.co/integrations/sophos[Sophos] Elastic integration instead. {pull}38037[38037] - Introduce input/netmetrics and refactor netflow input metrics {pull}38055[38055] - Update Salesforce module to use new Salesforce input. {pull}37509[37509] +- Tag events that come from a filestream in "take over" mode. {pull}39828[39828] - Fix high IO and handling of a corrupted registry log file. {pull}35893[35893] *Heartbeat* diff --git a/filebeat/docs/howto/migrate-to-filestream.asciidoc b/filebeat/docs/howto/migrate-to-filestream.asciidoc index a57105adb3e..30057fab725 100644 --- a/filebeat/docs/howto/migrate-to-filestream.asciidoc +++ b/filebeat/docs/howto/migrate-to-filestream.asciidoc @@ -247,3 +247,8 @@ and return to old `log` inputs the files that were taken by `filestream` inputs, 6. Run Filebeat with the old configuration (no `filestream` inputs with `take_over: true`). NOTE: Reverting to backups might cause some events to repeat, depends on the amount of time the new configuration was running. + +=== Debugging on Kibana + +Events produced by `filestream` with `take_over: true` contains `take_over` tag. +You can filter on this tag in Kibana and see the events which came from a filestream in the "take over" mode. \ No newline at end of file diff --git a/filebeat/input/filestream/input.go b/filebeat/input/filestream/input.go index 0136b062b48..7da25654a25 100644 --- a/filebeat/input/filestream/input.go +++ b/filebeat/input/filestream/input.go @@ -41,6 +41,7 @@ import ( "github.com/elastic/beats/v7/libbeat/reader/readfile/encoding" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) const pluginName = "filestream" @@ -61,6 +62,7 @@ type filestream struct { encodingFactory encoding.EncodingFactory closerConfig closerConfig parsers parser.Config + takeOver bool } // Plugin creates a new filestream input plugin for creating a stateful input. @@ -101,6 +103,7 @@ func configure(cfg *conf.C) (loginp.Prospector, loginp.Harvester, error) { encodingFactory: encodingFactory, closerConfig: config.Close, parsers: config.Reader.Parsers, + takeOver: config.TakeOver, } return prospector, filestream, nil @@ -378,6 +381,11 @@ func (inp *filestream) readFromSource( metrics.BytesProcessed.Add(uint64(message.Bytes)) + // add "take_over" tag if `take_over` is set to true + if inp.takeOver { + _ = mapstr.AddTags(message.Fields, []string{"take_over"}) + } + if err := p.Publish(message.ToEvent(), s); err != nil { metrics.ProcessingErrors.Inc() return err diff --git a/filebeat/input/filestream/input_test.go b/filebeat/input/filestream/input_test.go index a1d9729c5aa..3dfe176ac01 100644 --- a/filebeat/input/filestream/input_test.go +++ b/filebeat/input/filestream/input_test.go @@ -35,6 +35,7 @@ import ( "github.com/elastic/beats/v7/libbeat/statestore/storetest" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" ) func BenchmarkFilestream(b *testing.B) { @@ -115,6 +116,48 @@ paths: }) } +func TestTakeOverTags(t *testing.T) { + testCases := []struct { + name string + takeOver bool + testFunc func(t *testing.T, event beat.Event) + }{ + { + name: "test-take_over-true", + takeOver: true, + testFunc: func(t *testing.T, event beat.Event) { + tags, err := event.GetValue("tags") + require.NoError(t, err) + require.Contains(t, tags, "take_over") + }, + }, + { + name: "test-take_over-false", + takeOver: false, + testFunc: func(t *testing.T, event beat.Event) { + _, err := event.GetValue("tags") + require.ErrorIs(t, err, mapstr.ErrKeyNotFound) + }, + }, + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + filename := generateFile(t, t.TempDir(), 5) + cfg := fmt.Sprintf(` +type: filestream +prospector.scanner.check_interval: 1s +take_over: %t +paths: + - %s`, testCase.takeOver, filename) + runner := createFilestreamTestRunner(context.Background(), t, testCase.name, cfg, 5, true) + events := runner(t) + for _, event := range events { + testCase.testFunc(t, event) + } + }) + } +} + // runFilestreamBenchmark runs the entire filestream input with the in-memory registry and the test pipeline. // `testID` must be unique for each test run // `cfg` must be a valid YAML string containing valid filestream configuration From bb898001f52042c72ab87651096cb63bee9a9e58 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Wed, 12 Jun 2024 13:21:39 +0300 Subject: [PATCH 2/9] Migrate github check for Filebeat and x-pack/filebeat to Buildkite pipeline (#39751) * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Update filebeat-pipeline.yml * Migrate check * Update filebeat-pipeline.yml * Update pipeline.xpack.filebeat.yml * Update pipeline.xpack.filebeat.yml * Update pipeline.xpack.filebeat.yml * Update --- .buildkite/filebeat/filebeat-pipeline.yml | 26 +++++++++++++++ .buildkite/x-pack/pipeline.xpack.filebeat.yml | 27 ++++++++++++++++ .github/workflows/check-filebeat.yml | 32 ------------------- 3 files changed, 53 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/check-filebeat.yml diff --git a/.buildkite/filebeat/filebeat-pipeline.yml b/.buildkite/filebeat/filebeat-pipeline.yml index 87f1925a372..849720d1e2d 100644 --- a/.buildkite/filebeat/filebeat-pipeline.yml +++ b/.buildkite/filebeat/filebeat-pipeline.yml @@ -27,6 +27,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "Filebeat Check/Update" + key: "filebeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C filebeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "filebeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "filebeat-check-update" + - group: "Filebeat Mandatory Tests" key: "filebeat-mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.filebeat.yml b/.buildkite/x-pack/pipeline.xpack.filebeat.yml index 1aa1c51ca5f..ce45e798239 100644 --- a/.buildkite/x-pack/pipeline.xpack.filebeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.filebeat.yml @@ -26,6 +26,33 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/filebeat Check/Update" + key: "x-pack-filebeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/filebeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:0.3" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack-filebeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-filebeat-check-update" + + - group: "x-pack/filebeat Mandatory Tests" key: "x-pack-filebeat-mandatory-tests" steps: diff --git a/.github/workflows/check-filebeat.yml b/.github/workflows/check-filebeat.yml deleted file mode 100644 index 930a04ec5e5..00000000000 --- a/.github/workflows/check-filebeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-filebeat - -on: - pull_request: - paths: - - '.github/workflows/check-filebeat.yml' - - 'filebeat/**' - - 'x-pack/filebeat/**' - -env: - BEAT_MODULE: 'filebeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libsystemd-dev - run: sudo apt-get install -y libsystemd-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From f8679db91a544f6439ce3d4b4c8f2098abd2fcf0 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:13:56 +0300 Subject: [PATCH 3/9] Migrate github checks to buildkite for packetbeat and x-pack/packetbeat (#39871) * update * update --- .buildkite/packetbeat/pipeline.packetbeat.yml | 26 +++++++++++++++ .../x-pack/pipeline.xpack.packetbeat.yml | 26 +++++++++++++++ .github/workflows/check-packetbeat.yml | 32 ------------------- 3 files changed, 52 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/check-packetbeat.yml diff --git a/.buildkite/packetbeat/pipeline.packetbeat.yml b/.buildkite/packetbeat/pipeline.packetbeat.yml index c9a5c9f06f2..40e5f0f4b6f 100644 --- a/.buildkite/packetbeat/pipeline.packetbeat.yml +++ b/.buildkite/packetbeat/pipeline.packetbeat.yml @@ -25,6 +25,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "Packetbeat Check/Update" + key: "packetbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C packetbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "packetbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "packetbeat-check-update" + - group: "packetbeat Mandatory Tests" key: "packetbeat-mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml index 542fee787c1..71027426992 100644 --- a/.buildkite/x-pack/pipeline.xpack.packetbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.packetbeat.yml @@ -27,6 +27,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/packetbeat Check/Update" + key: "x-pack-packetbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/packetbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/packetbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-packetbeat-check-update" + - group: "x-pack/packetbeat Mandatory Tests" key: "x-pack-packetbeat-mandatory-tests" diff --git a/.github/workflows/check-packetbeat.yml b/.github/workflows/check-packetbeat.yml deleted file mode 100644 index ba05b6c0160..00000000000 --- a/.github/workflows/check-packetbeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-packetbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'packetbeat/**' - - 'x-pack/packetbeat/**' - -env: - BEAT_MODULE: 'packetbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From 02a57f76489ac349d24fd2a36818559ff314a624 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:14:12 +0300 Subject: [PATCH 4/9] Migrate github checks to buildkite for libbeat and x-pack/libbeat (#39869) * Update * Update pipeline.xpack.libbeat.yml * Update pipeline.xpack.libbeat.yml * update --- .buildkite/libbeat/pipeline.libbeat.yml | 26 ++++++++++++++++ .buildkite/x-pack/pipeline.xpack.libbeat.yml | 26 ++++++++++++++++ .github/workflows/check-libbeat.yml | 32 -------------------- 3 files changed, 52 insertions(+), 32 deletions(-) delete mode 100644 .github/workflows/check-libbeat.yml diff --git a/.buildkite/libbeat/pipeline.libbeat.yml b/.buildkite/libbeat/pipeline.libbeat.yml index fd485279858..7ebe5d76f98 100644 --- a/.buildkite/libbeat/pipeline.libbeat.yml +++ b/.buildkite/libbeat/pipeline.libbeat.yml @@ -16,6 +16,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "libbeat Check/Update" + key: "libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "libbeat-check-update" + - group: "Mandatory Tests" key: "mandatory-tests" steps: diff --git a/.buildkite/x-pack/pipeline.xpack.libbeat.yml b/.buildkite/x-pack/pipeline.xpack.libbeat.yml index 33356c78c46..93f57713ee3 100644 --- a/.buildkite/x-pack/pipeline.xpack.libbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.libbeat.yml @@ -24,6 +24,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/libbeat Check/Update" + key: "x-pack-libbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/libbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/libbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-libbeat-check-update" + - group: "x-pack/libbeat Mandatory Tests" key: "x-pack-libbeat-mandatory-tests" diff --git a/.github/workflows/check-libbeat.yml b/.github/workflows/check-libbeat.yml deleted file mode 100644 index 27e03701b85..00000000000 --- a/.github/workflows/check-libbeat.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: check-libbeat - -on: - pull_request: - paths: - - '.github/workflows/check-packetbeat.yml' - - 'libbeat/**' - - 'x-pack/libbeat/**' - -env: - BEAT_MODULE: 'libbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Update package lists - run: sudo apt-get update - - name: Install libpcap-dev - run: sudo apt-get install -y libpcap-dev - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes From 8bcf0babf2a7fd397520858d0a7712d4d3d5fd31 Mon Sep 17 00:00:00 2001 From: Alexandros Sapranidis Date: Wed, 12 Jun 2024 16:27:48 +0300 Subject: [PATCH 5/9] Allow github-actions bot to trigger builds (#39874) This commit adds GitHub-actions to the list of bots that are allowed to trigger BK builds. Signed-off-by: Alexandros Sapranidis --- .buildkite/pull-requests.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 55affae4128..b2cbb06e70f 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -5,7 +5,7 @@ "pipelineSlug": "beats", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, @@ -21,7 +21,7 @@ "pipelineSlug": "beats-xpack-elastic-agent", "allow_org_users": true, "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]"], + "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]"], "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, From 60ba7bb46d0396f71ae54c9d3cd0979f3919cf5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tanja=20Mili=C4=8Di=C4=87?= <156105538+tanja-milicic@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:02:27 +0200 Subject: [PATCH 6/9] Update opendashboards.asciidoc to rename index pattern to data view (#39827) --- libbeat/docs/shared/opendashboards.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/docs/shared/opendashboards.asciidoc b/libbeat/docs/shared/opendashboards.asciidoc index 7e73fbc8f4b..1b942504998 100644 --- a/libbeat/docs/shared/opendashboards.asciidoc +++ b/libbeat/docs/shared/opendashboards.asciidoc @@ -13,7 +13,7 @@ include::{libbeat-dir}/tab-widgets/open-kibana-widget.asciidoc[] -- . In the side navigation, click *Discover*. To see {beatname_uc} data, make -sure the predefined +{beatname_lc}-*+ index pattern is selected. +sure the predefined +{beatname_lc}-*+ data view is selected. + -- TIP: If you don’t see data in {kib}, try changing the time filter to a larger From 09707a2a279e7b8c0a55f3b19ac76d3754013140 Mon Sep 17 00:00:00 2001 From: VihasMakwana <121151420+VihasMakwana@users.noreply.github.com> Date: Wed, 12 Jun 2024 20:46:20 +0530 Subject: [PATCH 7/9] [docs][filebeat]: add 4th step in migrate-to-filestream (#39863) * docs: add 4th step in migrate-to-filestream * Update filebeat/docs/howto/migrate-to-filestream.asciidoc Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --------- Co-authored-by: Pierre HILBERT Co-authored-by: David Kilfoyle <41695641+kilfoyle@users.noreply.github.com> --- filebeat/docs/howto/migrate-to-filestream.asciidoc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/filebeat/docs/howto/migrate-to-filestream.asciidoc b/filebeat/docs/howto/migrate-to-filestream.asciidoc index 30057fab725..55c05a5b46d 100644 --- a/filebeat/docs/howto/migrate-to-filestream.asciidoc +++ b/filebeat/docs/howto/migrate-to-filestream.asciidoc @@ -234,6 +234,13 @@ The example configuration shown earlier needs to be adjusted as well: |backoff.max |=== +=== Step 4 + +The events produced by `filestream` input with `take_over: true` contain a `take_over` tag. +You can filter on this tag in Kibana and see the events which came from a filestream in the "take_over" mode. + +Once you start receiving events with this tag, you can remove `take_over: true` and restart the fileinput again. + === If something went wrong If for whatever reason you'd like to revert the configuration after running the migrated configuration From dc876a0a1dfb89c23a69a9f2fac01d2f55301cae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 12 Jun 2024 16:55:31 +0000 Subject: [PATCH 8/9] [updatecli] update elastic stack version for testing 8.15.0-04e5793a (#39670) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli * chore: Update snapshot.yml Made with ❤️️ by updatecli --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Denis --- testing/environments/snapshot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/environments/snapshot.yml b/testing/environments/snapshot.yml index 5b6dd6ffd00..a5fb79fb02c 100644 --- a/testing/environments/snapshot.yml +++ b/testing/environments/snapshot.yml @@ -3,7 +3,7 @@ version: '2.3' services: elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/elasticsearch/elasticsearch:8.15.0-d2070ed2-SNAPSHOT # When extend is used it merges healthcheck.tests, see: # https://github.com/docker/compose/issues/8962 # healthcheck: @@ -31,7 +31,7 @@ services: - "./docker/elasticsearch/users_roles:/usr/share/elasticsearch/config/users_roles" logstash: - image: docker.elastic.co/logstash/logstash:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/logstash/logstash:8.15.0-d2070ed2-SNAPSHOT healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"] retries: 600 @@ -44,7 +44,7 @@ services: - 5055:5055 kibana: - image: docker.elastic.co/kibana/kibana:8.15.0-9bce1e63-SNAPSHOT + image: docker.elastic.co/kibana/kibana:8.15.0-d2070ed2-SNAPSHOT environment: - "ELASTICSEARCH_USERNAME=kibana_system_user" - "ELASTICSEARCH_PASSWORD=testing" From 9e2ef861d608d9299985a87569229207dc4a3bd3 Mon Sep 17 00:00:00 2001 From: ev1yehor <146825775+ev1yehor@users.noreply.github.com> Date: Thu, 13 Jun 2024 00:44:25 +0300 Subject: [PATCH 9/9] Update (#39879) --- .buildkite/winlogbeat/pipeline.winlogbeat.yml | 26 +++++++++++++++++ .../x-pack/pipeline.xpack.winlogbeat.yml | 26 +++++++++++++++++ .github/workflows/check-winlogbeat.yml | 28 ------------------- 3 files changed, 52 insertions(+), 28 deletions(-) delete mode 100644 .github/workflows/check-winlogbeat.yml diff --git a/.buildkite/winlogbeat/pipeline.winlogbeat.yml b/.buildkite/winlogbeat/pipeline.winlogbeat.yml index c598224438f..4d8e468595d 100644 --- a/.buildkite/winlogbeat/pipeline.winlogbeat.yml +++ b/.buildkite/winlogbeat/pipeline.winlogbeat.yml @@ -23,6 +23,32 @@ env: DOCKER_PULL: 0 steps: + - group: "Winlogbeat Check/Update" + key: "winlogbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C winlogbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "winlogbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "winlogbeat-check-update" + - group: "Winlogbeat Mandatory Tests" key: "winlogbeat-mandatory-tests" diff --git a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml index 401517960aa..fc8ce29a407 100644 --- a/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml +++ b/.buildkite/x-pack/pipeline.xpack.winlogbeat.yml @@ -19,6 +19,32 @@ env: TEST_COVERAGE: "true" steps: + - group: "x-pack/winlogbeat Check/Update" + key: "x-pack-winlogbeat-check-update" + steps: + - label: "Run check/update" + command: | + make -C x-pack/winlogbeat check update + make check-no-changes + retry: + automatic: + - limit: 3 + agents: + image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest" + cpu: "4000m" + memory: "8Gi" + useCustomGlobalHooks: true + notify: + - github_commit_status: + context: "x-pack/winlogbeat: check/update" + + - wait: ~ + # with PRs, we want to run mandatory tests only if check/update step succeed + # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests + # this allows building DRA artifacts even if there is flakiness in check/update step + if: build.env("BUILDKITE_PULL_REQUEST") != "false" + depends_on: "x-pack-winlogbeat-check-update" + - group: "x-pack/winlogbeat Mandatory Tests" key: "x-pack-winlogbeat-mandatory-tests" steps: diff --git a/.github/workflows/check-winlogbeat.yml b/.github/workflows/check-winlogbeat.yml deleted file mode 100644 index a79c4bef209..00000000000 --- a/.github/workflows/check-winlogbeat.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: check-winlogbeat - -on: - pull_request: - paths: - - '.github/workflows/check-winlogbeat.yml' - - 'winlogbeat/**' - - 'x-pack/winlogbeat/**' - -env: - BEAT_MODULE: 'winlogbeat' - -permissions: - contents: read - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 - with: - go-version-file: .go-version - - name: Run check/update - run: | - go install github.com/magefile/mage - make -C ${{ env.BEAT_MODULE }} check update - make check-no-changes