From f7082769e5f3eed0cee2575544ba70f17650ebbb Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Tue, 15 Dec 2020 07:58:20 -0500 Subject: [PATCH] Add sort to system test query (#203) While executing system tests multiple times I was getting different results on each run. The root cause was that the test samples 10 of the events that are written to ES. And on each run it was getting different events back, validating them, and returning a unique set of errors. I changed the query to sort on the @timestamp and to also sample more events (up to 500). This way we get more repeatable tests. --- docs/howto/system_testing.md | 9 +++++++-- internal/testrunner/runners/system/runner.go | 5 +++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 23a0fa051..1f8d634db 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -12,7 +12,12 @@ Conceptually, running a system test involves the following steps: 1. Depending on the Elastic Package whose data stream is being tested, deploy an instance of the package's integration service. 1. Create a test policy that configures a single data stream for a single package. 1. Assign the test policy to the enrolled Agent. -1. Wait a reasonable amount of time for the Agent to collect data from the integration service and index it into the correct Elasticsearch data stream. +1. Wait a reasonable amount of time for the Agent to collect data from the + integration service and index it into the correct Elasticsearch data stream. +1. Query the first 500 documents based on `@timestamp` for validation. +1. Validate mappings are defined for the fields contained in the indexed documents. +1. Validate that the JSON data types contained `_source` are compatible with + mappings declared for the field. 1. Delete test artifacts and tear down the instance of the package's integration service. 1. Once all desired data streams have been system tested, tear down the Elastic Stack. @@ -20,7 +25,7 @@ Conceptually, running a system test involves the following steps: At the moment system tests have limitations. The salient ones are: * They can only test packages whose integration services can be deployed via Docker Compose. Eventually they will be able to test packages that can be deployed via other means, e.g. a Terraform configuration. -* They can only check for the _existence_ of data in the correct Elasticsearch data stream. Eventually they will be able to test the shape and contents of the indexed data as well. +* There isn't a way to do assert that the indexed data matches data from a file (e.g. golden file testing). ## Defining a system test diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 9c95dacd4..dccc0a062 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -33,6 +33,9 @@ func init() { const ( // TestType defining system tests TestType testrunner.TestType = "system" + + // Maximum number of events to query. + elasticsearchQuerySize = 500 ) type runner struct { @@ -270,6 +273,8 @@ func (r *runner) run() ([]testrunner.TestResult, error) { passed, err := waitUntilTrue(func() (bool, error) { resp, err := r.options.ESClient.Search( r.options.ESClient.Search.WithIndex(dataStream), + r.options.ESClient.Search.WithSort("@timestamp:asc"), + r.options.ESClient.Search.WithSize(elasticsearchQuerySize), ) if err != nil { return false, errors.Wrap(err, "could not search data stream")