ES package log pipelines

packages/elasticsearch/_dev/deploy/docker/config/elasticsearch.yml

@@ -0,0 +1,10 @@
+network.host: ""
+transport.host: "127.0.0.1"
+http.host: "0.0.0.0"
+indices.id_field_data.enabled: true
+xpack.license.self_generated.type: "trial"
+xpack.security.enabled: true
+xpack.security.authc.api_key.enabled: true
+xpack.security.http.ssl.enabled: false
+ingest.geoip.downloader.enabled: false
+xpack.security.audit.enabled: true

packages/elasticsearch/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,26 @@
+version: '2.3'
+services:
+  elasticsearch:
+    environment:
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - "ELASTIC_PASSWORD=changeme"
+    image: "docker.elastic.co/elasticsearch/elasticsearch:8.5.0-SNAPSHOT"
+    healthcheck:
+      test: ["CMD", "curl", "-f", "-u", "elastic:changeme", "http://127.0.0.1:9200/"]
+      retries: 300
+      interval: 1s
+    volumes:
+      - ./config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
+      - ./config/log4j2.properties:/usr/share/elasticsearch/config/log4j2.properties
+      - ${SERVICE_LOGS_DIR}:/usr/share/elasticsearch/logs
+    ports:
+      - "127.0.0.1:9201:9200"
+
+  logs_generation:
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    image: "alpine/curl:latest"
+    command: "./generate-logs.sh"
+    volumes:
+      - ./scripts/generate-logs.sh:/generate-logs.sh

packages/elasticsearch/_dev/deploy/docker/scripts/generate-logs.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+# Sends queries to the elasticsearch service configured in _dev/deploy in order
+# to generate all existing log types. `server` and `gc` logs will be generated
+# without external trigger.
+
+set -e
+
+elasticsearch_host=http://elasticsearch:9200
+username=elastic
+password=changeme
+
+# create an index that will trace every indexing/searching operations
+echo Creating foo-* index template with 0ms slowlog threshold
+curl -s -S -u $username:$password -X PUT $elasticsearch_host/_template/foo-template \
+  -H "Content-Type: application/json" \
+  -d "{\"index_patterns\": [\"foo-*\"], \"settings\": { \"index.indexing.slowlog.threshold.index.trace\": \"0ms\", \"index.search.slowlog.threshold.query.trace\": \"0ms\" } }"
+
+echo Creating foo-bar index
+curl -s -S -u $username:$password -X PUT $elasticsearch_host/foo-bar
+
+while true
+do
+  echo Generating audit, deprecation and slowlogs
+
+  # audit logs will be generated automatically on requests
+
+  # generates deprecation log and index_search slowlog
+  curl -s -S -u $username:$password -X POST $elasticsearch_host/foo-bar/_search \
+    -H "Content-Type: application/json" \
+    -d "{\"_source\": { \"exclude\": [\"bar\"] } }"
+
+  # generates index_indexing slowlog
+  curl -s -S -u $username:$password -X POST $elasticsearch_host/foo-bar/_doc \
+    -H "Content-Type: application/json" \
+    -d "{ \"foo\": \"bar\" }"
+
+  sleep 5
+done

packages/elasticsearch/_dev/deploy/docker/scripts/install-package.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Installs the elasticsearch package and attach an integration to the running Agent.
+# The package policy (./package-policy.json) we attach to the agent policy has
+# quite a few hardcoded values that we could potentially generate dynamically. The
+# most important one is the agent policy. We're currently relying on the Agent installed
+# by the `elastic-package stack` command but ideally we'd create a dedicated, standalone
+# agent that we can fully control.
+
+set -e
+
+eval "$(elastic-package stack shellinit)"
+
+script_dir=$(dirname ${BASH_SOURCE[0]})
+
+package_version=`cat $script_dir/../../../../manifest.yml | sed -nr 's/^version: (.*)$/\1/p'`
+es_service_container=`docker ps --filter "publish=9201" --format "{{print .Names}}"`
+elastic_agent_logs_dir=/tmp/service_logs
+policy_name=script-generated-elasticsearch-policy
+package_policy_file=$script_dir/package-policy.json
+kibana_host=$ELASTIC_PACKAGE_KIBANA_HOST
+
+# these are the values of the es service started by `elastic-package service up`
+elasticsearch_host=http://$es_service_container:9200
+username=elastic
+password=changeme
+
+if [[ -z $es_service_container ]]; then
+  echo No elasticsearch service found. Run "elastic-package service up -v" in the elasticsearch package to start the service
+  exit 1
+fi
+
+echo Found elasticsearch service "$es_service_container"
+
+elastic-package install
+
+policy=`cat $package_policy_file |
+  sed "s~__PACKAGE_VERSION__~$package_version~g" |
+  sed "s~__INTEGRATION_POLICY_NAME__~$policy_name~g" |
+  sed "s~__ELASTICSEARCH_HOST__~$elasticsearch_host~g" |
+  sed "s~__ELASTICSEARCH_USERNAME__~$username~g" |
+  sed "s~__ELASTICSEARCH_PASSWORD__~$password~g" |
+  sed "s~__ELASTIC_AGENT_LOGS_DIR__~$elastic_agent_logs_dir~g"`
+
+echo Attaching elasticsearch to Agent policy
+
+curl --insecure -Ss -u $username:$password -X POST $kibana_host/api/fleet/package_policies \
+  -H "kbn-xsrf: 1" \
+  -H "Content-Type: application/json" \
+  -d "$policy"

packages/elasticsearch/_dev/deploy/docker/scripts/package-policy.json

@@ -0,0 +1,209 @@
+{
+  "name": "__INTEGRATION_POLICY_NAME__",
+  "description": "",
+  "namespace": "default",
+  "policy_id": "elastic-agent-managed-ep",
+  "enabled": true,
+  "output_id": "",
+  "inputs": [
+    {
+      "type": "logfile",
+      "policy_template": "elasticsearch",
+      "enabled": true,
+      "streams": [
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "logs",
+            "dataset": "elasticsearch.audit"
+          },
+          "vars": {
+            "paths": {
+              "value": [
+                "__ELASTIC_AGENT_LOGS_DIR__/*_audit.json"
+              ],
+              "type": "text"
+            }
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "logs",
+            "dataset": "elasticsearch.deprecation"
+          },
+          "vars": {
+            "paths": {
+              "value": [
+                "__ELASTIC_AGENT_LOGS_DIR__/*_deprecation.json"
+              ],
+              "type": "text"
+            }
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "logs",
+            "dataset": "elasticsearch.gc"
+          },
+          "vars": {
+            "paths": {
+              "value": [
+                "__ELASTIC_AGENT_LOGS_DIR__/gc.log.[0-9]*",
+                "__ELASTIC_AGENT_LOGS_DIR__/gc.log"
+              ],
+              "type": "text"
+            }
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "logs",
+            "dataset": "elasticsearch.server"
+          },
+          "vars": {
+            "paths": {
+              "value": [
+                "__ELASTIC_AGENT_LOGS_DIR__/*_server.json"
+              ],
+              "type": "text"
+            }
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "logs",
+            "dataset": "elasticsearch.slowlog"
+          },
+          "vars": {
+            "paths": {
+              "value": [
+                "__ELASTIC_AGENT_LOGS_DIR__/*_index_search_slowlog.json",
+                "__ELASTIC_AGENT_LOGS_DIR__/*_index_indexing_slowlog.json"
+              ],
+              "type": "text"
+            }
+          }
+        }
+      ]
+    },
+    {
+      "type": "elasticsearch/metrics",
+      "policy_template": "elasticsearch",
+      "enabled": true,
+      "streams": [
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.ccr"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.cluster_stats"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.enrich"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.index"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.index_recovery"
+          },
+          "vars": {
+            "active.only": {
+              "value": true,
+              "type": "bool"
+            }
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.index_summary"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.ml_job"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.node"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.node_stats"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.pending_tasks"
+          }
+        },
+        {
+          "enabled": true,
+          "data_stream": {
+            "type": "metrics",
+            "dataset": "elasticsearch.shard"
+          }
+        }
+      ],
+      "vars": {
+        "hosts": {
+          "value": [
+            "__ELASTICSEARCH_HOST__"
+          ],
+          "type": "text"
+        },
+        "username": {
+          "value": "__ELASTICSEARCH_USERNAME__",
+          "type": "text"
+        },
+        "password": {
+          "value": "__ELASTICSEARCH_PASSWORD__",
+          "type": "password"
+        },
+        "scope": {
+          "value": "node",
+          "type": "text"
+        }
+      }
+    }
+  ],
+  "package": {
+    "name": "elasticsearch",
+    "title": "Elasticsearch",
+    "version": "__PACKAGE_VERSION__"
+  }
+}