Add system tests to elasticsearch package

packages/elasticsearch/_dev/build/build.yml

@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@1.12
+    reference: git@8.4

packages/elasticsearch/_dev/deploy/docker/docker-compose.yml

@@ -1,21 +1,30 @@
-version: '2.3'
+version: "2.3"
 services:
   elasticsearch:
+    image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION:-8.5.0-SNAPSHOT}"
     environment:
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
       - "ELASTIC_PASSWORD=changeme"
-    image: "docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}"
     healthcheck:
-      test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://127.0.0.1:9200/"]
+      test:
+        [
+          "CMD",
+          "curl",
+          "-f",
+          "-u",
+          "elastic:changeme",
+          "http://127.0.0.1:9200/",
+        ]
       retries: 300
       interval: 1s
     volumes:
       - ./config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
       - ./config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties
-      - ${SERVICE_LOGS_DIR}:/usr/share/elasticsearch/logs
+      - es_logs:/usr/share/elasticsearch/logs
     ports:
       - "127.0.0.1:9201:9200"
   logs_generation:
+    user: root
     depends_on:
       elasticsearch:
         condition: service_healthy
@@ -27,3 +36,10 @@ services:
     command: "./generate-logs.sh"
     volumes:
       - ./scripts/generate-logs.sh:/generate-logs.sh
+      - "${SERVICE_LOGS_DIR}:/var/log"
+      - es_logs:/es_logs
+volumes:
+  es_logs:
+    driver_opts:
+      type: tmpfs
+      device: tmpfs

packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh

@@ -3,11 +3,26 @@
 # Sends queries to the elasticsearch service configured in _dev/deploy in order
 # to generate all existing log types. `server` and `gc` logs will be generated
 # without external trigger.
-
 set -e
 
 auth=$(echo -n $ES_SERVICE_USERNAME:$ES_SERVICE_PASSWORD | base64)
 
+# Copy the log files content from this container to /var/log/ which is a bind mounted to ${SERVICE_LOGS_DIR}
+# This sh must be executed by a root user in order to have permission to write in the ${SERVICE_LOGS_DIR} folder
+copy_log_files () {
+  for f in /es_logs/*;
+  do
+    echo "Copy ${f##*/} file..."
+
+    if [[ ! -e /var/log/${f##*/} ]]; then
+      touch /var/log/${f##*/}
+    fi
+
+    ## appends only new lines
+    comm -23 "$f" /var/log/${f##*/} >> /var/log/${f##*/}
+  done
+}
+
 # create an index that will trace every indexing/searching operations
 curl --request PUT \
   --url $ES_SERVICE_HOST/test_1 \
@@ -32,6 +47,72 @@ curl --request PUT \
 }
 }'
 
+# set machine learning job
+curl --request PUT \
+  --url $ES_SERVICE_HOST/_ml/anomaly_detectors/test-job1?pretty \
+  --header "Authorization: Basic $auth" \
+  --header 'Content-Type: application/json' \
+  --header 'X-Opaque-ID: myApp1' \
+  --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01' \
+  --data '{
+            "analysis_config": {
+              "bucket_span": "15m",
+              "detectors": [
+                {
+                  "detector_description": "Sum of bytes",
+                  "function": "sum",
+                  "field_name": "bytes"
+                }
+              ]
+            },
+            "data_description": {
+              "time_field": "timestamp",
+              "time_format": "epoch_ms"
+            },
+            "analysis_limits": {
+              "model_memory_limit": "11MB"
+            },
+            "model_plot_config": {
+              "enabled": true,
+              "annotations_enabled": true
+            },
+            "results_index_name": "test-job1",
+            "datafeed_config":
+            {
+              "indices": [
+                "kibana_sample_data_logs"
+              ],
+              "query": {
+                "bool": {
+                  "must": [
+                    {
+                      "match_all": {}
+                    }
+                  ]
+                }
+              },
+              "runtime_mappings": {
+                "hour_of_day": {
+                  "type": "long",
+                  "script": {
+                    "source": "emit(doc['timestamp'].value.getHour());"
+                  }
+                }
+              },
+              "datafeed_id": "datafeed-test-job1"
+            }
+          }'
+
+## Open ML job
+curl --request POST \
+  --url $ES_SERVICE_HOST/_ml/anomaly_detectors/test-job1/_open \
+  --header "Authorization: Basic $auth" \
+  --header 'Content-Type: application/json' \
+  --header 'X-Opaque-ID: myApp1' \
+  --header 'traceparent: 00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01'
+
+
+
 while true
 do
   echo Generating slowlogs, audit and deprecation
@@ -125,5 +206,7 @@ do
       }
   }'
 
+  copy_log_files
+
   sleep 10
 done

packages/elasticsearch/_dev/deploy/variants.yml

@@ -0,0 +1,4 @@
+variants:
+  elasticsearch_8.5.0:
+    ELASTIC_VERSION: 8.5.0-SNAPSHOT
+default: elasticsearch_8.5.0

packages/elasticsearch/data_stream/audit/_dev/test/system/test-default-config.yml

@@ -0,0 +1,5 @@
+input: logfile
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/*_audit.json"

packages/elasticsearch/data_stream/audit/sample_event.json

@@ -1,8 +1,8 @@
 {
-    "@timestamp": "2022-09-01T19:20:17.967Z",
+    "@timestamp": "2022-10-11T11:47:42.250Z",
     "agent": {
-        "ephemeral_id": "ec83bfa3-8e61-430e-91ca-dc784bfa56c0",
-        "id": "97025ce1-28a3-4aeb-926b-ed68301fc4d2",
+        "ephemeral_id": "5fd89ef0-54ad-4721-a07d-fd8ba72e7dc0",
+        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
         "name": "docker-fleet-agent",
         "type": "filebeat",
         "version": "8.5.0"
@@ -16,7 +16,7 @@
         "version": "1.10.0"
     },
     "elastic_agent": {
-        "id": "97025ce1-28a3-4aeb-926b-ed68301fc4d2",
+        "id": "79e48fe3-2ecd-4021-aed5-6e7e69d47606",
         "snapshot": true,
         "version": "8.5.0"
     },
@@ -30,7 +30,7 @@
             "origin": {},
             "origin.type": "rest",
             "request": {
-                "id": "YCHBXylbRnSC3Vc8-f3sIA"
+                "id": "grN8d7FlRZ-sz1j67QPBpA"
             },
             "request.name": "MainRequest",
             "user": {},
@@ -40,47 +40,47 @@
             ]
         },
         "cluster": {
-            "uuid": "wkVNYOctQ8mbbp1EkrFjKw"
+            "uuid": "wrIdKwZLQAS1C_yXW4FEWQ"
         },
         "node": {
-            "id": "VdwTr-luTomz8dDpOp2OJQ"
+            "id": "GZVRKUAoSlmHOSMKjN5KyQ"
         }
     },
     "event": {
         "action": "access_granted",
         "agent_id_status": "verified",
         "category": "database",
-        "created": "2022-09-01T19:20:39.899Z",
+        "created": "2022-10-11T11:47:57.695Z",
         "dataset": "elasticsearch.audit",
-        "ingested": "2022-09-01T19:20:43Z",
+        "ingested": "2022-10-11T11:47:58Z",
         "kind": "event",
         "outcome": "success"
     },
     "host": {
         "architecture": "x86_64",
-        "containerized": true,
+        "containerized": false,
         "hostname": "docker-fleet-agent",
-        "id": "VdwTr-luTomz8dDpOp2OJQ",
+        "id": "GZVRKUAoSlmHOSMKjN5KyQ",
         "ip": [
-            "172.21.0.7"
+            "192.168.0.7"
         ],
         "mac": [
-            "02:42:ac:15:00:07"
+            "02-42-C0-A8-00-07"
         ],
         "name": "docker-fleet-agent",
         "os": {
             "codename": "focal",
             "family": "debian",
-            "kernel": "5.10.47-linuxkit",
+            "kernel": "5.10.124-linuxkit",
             "name": "Ubuntu",
             "platform": "ubuntu",
             "type": "linux",
-            "version": "20.04.4 LTS (Focal Fossa)"
+            "version": "20.04.5 LTS (Focal Fossa)"
         }
     },
     "http": {
         "request": {
-            "id": "YCHBXylbRnSC3Vc8-f3sIA"
+            "id": "grN8d7FlRZ-sz1j67QPBpA"
         }
     },
     "input": {
@@ -93,7 +93,7 @@
         "level": "info",
         "offset": 0
     },
-    "message": "{\"type\":\"audit\", \"timestamp\":\"2022-09-01T19:20:17,967+0000\", \"cluster.uuid\":\"wkVNYOctQ8mbbp1EkrFjKw\", \"node.id\":\"VdwTr-luTomz8dDpOp2OJQ\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:53716\", \"request.id\":\"YCHBXylbRnSC3Vc8-f3sIA\", \"action\":\"cluster:monitor/main\", \"request.name\":\"MainRequest\"}",
+    "message": "{\"type\":\"audit\", \"timestamp\":\"2022-10-11T11:47:42,250+0000\", \"cluster.uuid\":\"wrIdKwZLQAS1C_yXW4FEWQ\", \"node.id\":\"GZVRKUAoSlmHOSMKjN5KyQ\", \"event.type\":\"transport\", \"event.action\":\"access_granted\", \"authentication.type\":\"REALM\", \"user.name\":\"elastic\", \"user.realm\":\"reserved\", \"user.roles\":[\"superuser\"], \"origin.type\":\"rest\", \"origin.address\":\"127.0.0.1:48692\", \"request.id\":\"grN8d7FlRZ-sz1j67QPBpA\", \"action\":\"cluster:monitor/main\", \"request.name\":\"MainRequest\"}",
     "related": {
         "user": [
             "elastic"
@@ -103,9 +103,9 @@
         "type": "elasticsearch"
     },
     "source": {
-        "address": "127.0.0.1:53716",
+        "address": "127.0.0.1:48692",
         "ip": "127.0.0.1",
-        "port": 53716
+        "port": 48692
     },
     "user": {
         "name": "elastic"

packages/elasticsearch/data_stream/cluster_stats/_dev/test/system/test-default-config.yml

@@ -0,0 +1,11 @@
+type: elasticsearch/metrics
+dataset: elasticsearch.stack_monitoring.cluster_stats
+vars:
+  hosts:
+    - "http://{{Hostname}}:9200"
+  username: elastic
+  password: changeme
+data_stream: ~
+numeric_keyword_fields:
+  - elasticsearch.version
+  - elasticsearch.cluster.stats.state.nodes_hash

packages/elasticsearch/data_stream/cluster_stats/fields/fields.yml

@@ -15,6 +15,8 @@
           type: keyword
         - name: state_uuid
           type: keyword
+        - name: nodes
+          type: flattened
     - name: status
       type: keyword
       description: Cluster status (green, yellow, red).
@@ -45,6 +47,8 @@
               type: long
             - name: memory.heap.used.bytes
               type: long
+        - name: versions
+          type: text
     - name: indices
       type: group
       fields:
@@ -74,12 +78,30 @@
     - name: license
       type: group
       fields:
+        - name: cluster_needs_tls
+          type: boolean
+        - name: expiry_date
+          type: date
         - name: expiry_date_in_millis
           type: long
+        - name: issue_date
+          type: date
+        - name: issue_date_in_millis
+          type: long
+        - name: issued_to
+          type: keyword
+        - name: issuer
+          type: keyword
+        - name: max_nodes
+          type: long
+        - name: start_date_in_millis
+          type: long
         - name: status
           type: keyword
         - name: type
           type: keyword
+        - name: uid
+          type: keyword
     - name: stack
       type: group
       fields: