[7.x] [Telemetry] Collector Schema (#64942) (#70141)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

.github/CODEOWNERS

@@ -0,0 +1,244 @@
+# GitHub CODEOWNERS definition
+# Identify which groups will be pinged by changes to different parts of the codebase.
+# For more info, see https://help.github.com/articles/about-codeowners/
+
+# App
+/x-pack/plugins/dashboard_enhanced/ @elastic/kibana-app
+/x-pack/plugins/discover_enhanced/ @elastic/kibana-app
+/x-pack/plugins/lens/ @elastic/kibana-app
+/x-pack/plugins/graph/ @elastic/kibana-app
+/src/legacy/core_plugins/kibana/public/local_application_service/ @elastic/kibana-app
+/src/plugins/dashboard/ @elastic/kibana-app
+/src/plugins/discover/ @elastic/kibana-app
+/src/plugins/input_control_vis/ @elastic/kibana-app
+/src/plugins/kibana_legacy/ @elastic/kibana-app
+/src/plugins/vis_default_editor/ @elastic/kibana-app
+/src/plugins/vis_type_markdown/ @elastic/kibana-app
+/src/plugins/vis_type_metric/ @elastic/kibana-app
+/src/plugins/vis_type_table/ @elastic/kibana-app
+/src/plugins/vis_type_tagcloud/ @elastic/kibana-app
+/src/plugins/vis_type_timelion/ @elastic/kibana-app
+/src/plugins/vis_type_timeseries/ @elastic/kibana-app
+/src/plugins/vis_type_vega/ @elastic/kibana-app
+/src/plugins/vis_type_vislib/ @elastic/kibana-app
+/src/plugins/vis_type_xy/ @elastic/kibana-app
+/src/plugins/visualize/ @elastic/kibana-app
+
+# Core UI
+# Exclude tutorials folder for now because they are not owned by Kibana app and most will move out soon
+/src/plugins/home/public @elastic/kibana-core-ui
+/src/plugins/home/server/*.ts @elastic/kibana-core-ui
+/src/plugins/home/server/services/ @elastic/kibana-core-ui
+# Exclude tutorial resources folder for now because they are not owned by Kibana app and most will move out soon
+/src/legacy/core_plugins/kibana/public/home/*.ts @elastic/kibana-core-ui
+/src/legacy/core_plugins/kibana/public/home/*.scss @elastic/kibana-core-ui
+/src/legacy/core_plugins/kibana/public/home/np_ready/ @elastic/kibana-core-ui
+
+# App Architecture
+/examples/developer_examples/ @elastic/kibana-app-arch
+/examples/url_generators_examples/ @elastic/kibana-app-arch
+/examples/url_generators_explorer/ @elastic/kibana-app-arch
+/packages/kbn-interpreter/ @elastic/kibana-app-arch
+/packages/elastic-datemath/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/embeddable_api/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/interpreter/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana_react/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana/public/management/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/kibana/server/routes/api/management/ @elastic/kibana-app-arch
+/src/legacy/core_plugins/visualizations/ @elastic/kibana-app-arch
+/src/legacy/server/index_patterns/ @elastic/kibana-app-arch
+/src/plugins/advanced_settings/ @elastic/kibana-app-arch
+/src/plugins/bfetch/ @elastic/kibana-app-arch
+/src/plugins/data/ @elastic/kibana-app-arch
+/src/plugins/embeddable/ @elastic/kibana-app-arch
+/src/plugins/expressions/ @elastic/kibana-app-arch
+/src/plugins/inspector/ @elastic/kibana-app-arch
+/src/plugins/kibana_react/ @elastic/kibana-app-arch
+/src/plugins/kibana_react/public/code_editor @elastic/kibana-canvas
+/src/plugins/kibana_utils/ @elastic/kibana-app-arch
+/src/plugins/management/ @elastic/kibana-app-arch
+/src/plugins/navigation/ @elastic/kibana-app-arch
+/src/plugins/share/ @elastic/kibana-app-arch
+/src/plugins/ui_actions/ @elastic/kibana-app-arch
+/src/plugins/visualizations/ @elastic/kibana-app-arch
+/x-pack/plugins/advanced_ui_actions/ @elastic/kibana-app-arch
+/x-pack/plugins/data_enhanced/ @elastic/kibana-app-arch
+/x-pack/plugins/drilldowns/ @elastic/kibana-app-arch
+
+# APM
+/x-pack/plugins/apm/  @elastic/apm-ui
+/x-pack/test/functional/apps/apm/  @elastic/apm-ui
+/src/legacy/core_plugins/apm_oss/  @elastic/apm-ui
+/src/plugins/apm_oss/  @elastic/apm-ui
+/src/apm.js @watson @vigneshshanmugam
+
+# Beats
+/x-pack/legacy/plugins/beats_management/  @elastic/beats
+
+# Canvas
+/x-pack/plugins/canvas/  @elastic/kibana-canvas
+/x-pack/test/functional/apps/canvas/  @elastic/kibana-canvas
+
+# Observability UIs
+/x-pack/legacy/plugins/infra/ @elastic/logs-metrics-ui
+/x-pack/plugins/infra/ @elastic/logs-metrics-ui
+/x-pack/plugins/ingest_manager/ @elastic/ingest-management
+/x-pack/legacy/plugins/ingest_manager/ @elastic/ingest-management
+/x-pack/plugins/observability/ @elastic/logs-metrics-ui @elastic/apm-ui @elastic/uptime @elastic/ingest-management
+/x-pack/legacy/plugins/monitoring/ @elastic/stack-monitoring-ui
+/x-pack/plugins/monitoring/ @elastic/stack-monitoring-ui
+/x-pack/plugins/uptime @elastic/uptime
+
+# Machine Learning
+/x-pack/legacy/plugins/ml/  @elastic/ml-ui
+/x-pack/plugins/ml/  @elastic/ml-ui
+/x-pack/test/functional/apps/machine_learning/  @elastic/ml-ui
+/x-pack/test/functional/services/machine_learning/  @elastic/ml-ui
+/x-pack/test/functional/services/ml.ts  @elastic/ml-ui
+# ML team owns and maintains the transform plugin despite it living in the Elasticsearch management section.
+/x-pack/plugins/transform/  @elastic/ml-ui
+/x-pack/test/functional/apps/transform/ @elastic/ml-ui
+/x-pack/test/functional/services/transform_ui/ @elastic/ml-ui
+/x-pack/test/functional/services/transform.ts @elastic/ml-ui
+
+# Maps
+/x-pack/legacy/plugins/maps/ @elastic/kibana-gis
+/x-pack/plugins/maps/ @elastic/kibana-gis
+/x-pack/test/api_integration/apis/maps/ @elastic/kibana-gis
+/x-pack/test/functional/apps/maps/ @elastic/kibana-gis
+/x-pack/test/functional/es_archives/maps/ @elastic/kibana-gis
+/x-pack/test/visual_regression/tests/maps/index.js @elastic/kibana-gis
+
+# Operations
+/src/dev/ @elastic/kibana-operations
+/src/setup_node_env/ @elastic/kibana-operations
+/src/optimize/ @elastic/kibana-operations
+/src/es_archiver/ @elastic/kibana-operations
+/packages/*eslint*/ @elastic/kibana-operations
+/packages/*babel*/ @elastic/kibana-operations
+/packages/kbn-dev-utils*/ @elastic/kibana-operations
+/packages/kbn-es/ @elastic/kibana-operations
+/packages/kbn-optimizer/ @elastic/kibana-operations
+/packages/kbn-pm/ @elastic/kibana-operations
+/packages/kbn-test/ @elastic/kibana-operations
+/packages/kbn-ui-shared-deps/ @elastic/kibana-operations
+/src/legacy/server/keystore/ @elastic/kibana-operations
+/src/legacy/server/pid/ @elastic/kibana-operations
+/src/legacy/server/sass/ @elastic/kibana-operations
+/src/legacy/server/utils/ @elastic/kibana-operations
+/src/legacy/server/warnings/ @elastic/kibana-operations
+/.ci/es-snapshots/ @elastic/kibana-operations
+/vars/ @elastic/kibana-operations
+
+# Quality Assurance
+/src/dev/code_coverage @elastic/kibana-qa
+/test/functional/services/common @elastic/kibana-qa
+/test/functional/services/lib @elastic/kibana-qa
+/test/functional/services/remote @elastic/kibana-qa
+
+# Platform
+/src/core/  @elastic/kibana-platform
+/config/kibana.yml @elastic/kibana-platform
+/x-pack/plugins/features/  @elastic/kibana-platform
+/x-pack/plugins/licensing/  @elastic/kibana-platform
+/x-pack/plugins/global_search/  @elastic/kibana-platform
+/x-pack/plugins/cloud/ @elastic/kibana-platform
+/packages/kbn-config-schema/ @elastic/kibana-platform
+/src/legacy/server/config/ @elastic/kibana-platform
+/src/legacy/server/http/ @elastic/kibana-platform
+/src/legacy/server/logging/ @elastic/kibana-platform
+/src/legacy/server/saved_objects/  @elastic/kibana-platform
+/src/legacy/server/status/ @elastic/kibana-platform
+/src/plugins/status_page/ @elastic/kibana-platform
+/src/plugins/saved_objects_management/ @elastic/kibana-platform
+/src/dev/run_check_published_api_changes.ts @elastic/kibana-platform
+
+# Security
+/src/core/server/csp/  @elastic/kibana-security @elastic/kibana-platform
+/x-pack/legacy/plugins/security/  @elastic/kibana-security
+/x-pack/legacy/plugins/spaces/  @elastic/kibana-security
+/x-pack/plugins/spaces/  @elastic/kibana-security
+/x-pack/plugins/encrypted_saved_objects/  @elastic/kibana-security
+/x-pack/plugins/security/  @elastic/kibana-security
+/x-pack/test/api_integration/apis/security/ @elastic/kibana-security
+
+# Kibana Localization
+/src/dev/i18n/  @elastic/kibana-localization
+/src/legacy/server/i18n/  @elastic/kibana-localization
+/src/core/public/i18n/  @elastic/kibana-localization
+/packages/kbn-i18n/  @elastic/kibana-localization
+
+# Kibana Telemetry
+/packages/kbn-analytics/ @elastic/kibana-telemetry
+/packages/kbn-telemetry-tools/ @elastic/kibana-telemetry
+/src/plugins/kibana_usage_collection/ @elastic/kibana-telemetry
+/src/plugins/newsfeed/ @elastic/kibana-telemetry
+/src/plugins/telemetry/ @elastic/kibana-telemetry
+/src/plugins/telemetry_collection_manager/ @elastic/kibana-telemetry
+/src/plugins/telemetry_management_section/ @elastic/kibana-telemetry
+/src/plugins/usage_collection/ @elastic/kibana-telemetry
+/x-pack/plugins/telemetry_collection_xpack/ @elastic/kibana-telemetry
+/.telemetryrc.json @elastic/kibana-telemetry
+/x-pack/.telemetryrc.json @elastic/kibana-telemetry
+src/plugins/telemetry/schema/legacy_oss_plugins.json @elastic/kibana-telemetry
+src/plugins/telemetry/schema/oss_plugins.json @elastic/kibana-telemetry
+x-pack/plugins/telemetry_collection_xpack/schema/xpack_plugins.json @elastic/kibana-telemetry
+
+# Kibana Alerting Services
+/x-pack/plugins/alerts/ @elastic/kibana-alerting-services
+/x-pack/plugins/actions/ @elastic/kibana-alerting-services
+/x-pack/plugins/event_log/ @elastic/kibana-alerting-services
+/x-pack/plugins/task_manager/ @elastic/kibana-alerting-services
+/x-pack/test/alerting_api_integration/ @elastic/kibana-alerting-services
+/x-pack/test/plugin_api_integration/plugins/task_manager/ @elastic/kibana-alerting-services
+/x-pack/test/plugin_api_integration/test_suites/task_manager/ @elastic/kibana-alerting-services
+/x-pack/plugins/triggers_actions_ui/ @elastic/kibana-alerting-services
+/x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/ @elastic/kibana-alerting-services
+/x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/ @elastic/kibana-alerting-services
+
+# Design
+**/*.scss  @elastic/kibana-design
+
+# Elasticsearch UI
+/src/plugins/dev_tools/ @elastic/es-ui
+/src/plugins/console/  @elastic/es-ui
+/src/plugins/es_ui_shared/  @elastic/es-ui
+/x-pack/legacy/plugins/cross_cluster_replication/  @elastic/es-ui
+/x-pack/plugins/index_lifecycle_management/  @elastic/es-ui
+/x-pack/legacy/plugins/index_management/  @elastic/es-ui
+/x-pack/legacy/plugins/license_management/  @elastic/es-ui
+/x-pack/legacy/plugins/rollup/  @elastic/es-ui
+/x-pack/legacy/plugins/snapshot_restore/  @elastic/es-ui
+/x-pack/legacy/plugins/upgrade_assistant/  @elastic/es-ui
+/x-pack/plugins/console_extensions/  @elastic/es-ui
+/x-pack/plugins/es_ui_shared/  @elastic/es-ui
+/x-pack/plugins/grokdebugger/  @elastic/es-ui
+/x-pack/plugins/index_management/  @elastic/es-ui
+/x-pack/plugins/license_management/  @elastic/es-ui
+/x-pack/plugins/painless_lab/  @elastic/es-ui
+/x-pack/plugins/remote_clusters/  @elastic/es-ui
+/x-pack/plugins/rollup/  @elastic/es-ui
+/x-pack/plugins/searchprofiler/  @elastic/es-ui
+/x-pack/plugins/snapshot_restore/  @elastic/es-ui
+/x-pack/plugins/upgrade_assistant/  @elastic/es-ui
+/x-pack/plugins/watcher/  @elastic/es-ui
+/x-pack/plugins/ingest_pipelines/  @elastic/es-ui
+
+# Endpoint
+/x-pack/plugins/endpoint/ @elastic/endpoint-app-team @elastic/siem
+/x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team @elastic/siem
+/x-pack/test/endpoint_api_integration_no_ingest/ @elastic/endpoint-app-team @elastic/siem
+/x-pack/test/security_solution_endpoint/ @elastic/endpoint-app-team @elastic/siem
+/x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team @elastic/siem
+/x-pack/test/plugin_functional/plugins/resolver_test/ @elastic/endpoint-app-team @elastic/siem
+/x-pack/test/plugin_functional/test_suites/resolver/ @elastic/endpoint-app-team @elastic/siem
+
+# Security Solution
+/x-pack/plugins/security_solution/ @elastic/siem @elastic/endpoint-app-team
+/x-pack/test/detection_engine_api_integration @elastic/siem @elastic/endpoint-app-team
+/x-pack/test/api_integration/apis/security_solution @elastic/siem @elastic/endpoint-app-team
+/x-pack/plugins/case @elastic/siem @elastic/endpoint-app-team
+/x-pack/plugins/lists @elastic/siem @elastic/endpoint-app-team
+
+# Security Intelligence And Analytics
+/x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics

.telemetryrc.json

@@ -0,0 +1,25 @@
+[
+  {
+    "output": "src/plugins/telemetry/schema/legacy_oss_plugins.json",
+    "root": "src/legacy/core_plugins/",
+    "exclude": [
+      "src/legacy/core_plugins/testbed",
+      "src/legacy/core_plugins/elasticsearch",
+      "src/legacy/core_plugins/tests_bundle"
+    ]
+  },
+  {
+    "output": "src/plugins/telemetry/schema/oss_plugins.json",
+    "root": "src/plugins/",
+    "exclude": [
+      "src/plugins/kibana_react/",
+      "src/plugins/testbed/",
+      "src/plugins/kibana_utils/",
+      "src/plugins/kibana_usage_collection/server/collectors/kibana/kibana_usage_collector.ts",
+      "src/plugins/kibana_usage_collection/server/collectors/application_usage/telemetry_application_usage_collector.ts",
+      "src/plugins/kibana_usage_collection/server/collectors/management/telemetry_management_collector.ts",
+      "src/plugins/kibana_usage_collection/server/collectors/ui_metric/telemetry_ui_metric_collector.ts",
+      "src/plugins/telemetry/server/collectors/usage/telemetry_usage_collector.ts"
+    ]
+  }
+]

package.json

@@ -136,6 +136,7 @@
     "@kbn/babel-preset": "1.0.0",
     "@kbn/config-schema": "1.0.0",
     "@kbn/i18n": "1.0.0",
+    "@kbn/telemetry-tools": "1.0.0",
     "@kbn/interpreter": "1.0.0",
     "@kbn/pm": "1.0.0",
     "@kbn/test-subj-selector": "0.2.1",

packages/kbn-telemetry-tools/README.md

@@ -0,0 +1,89 @@
+# Telemetry Tools
+
+## Schema extraction tool
+
+### Description
+
+The tool is used to extract telemetry collectors schema from all `*.{ts}` files in provided plugins directories to JSON files. The tool looks for `.telemetryrc.json` files in the root of the project and in the `x-pack` dir for its runtime configurations.
+
+It uses typescript parser to build an AST for each file. The tool is able to validate, extract and match collector schemas.
+
+### Examples and restrictions
+
+**Global restrictions**:
+
+The `id` can be only a string literal, it cannot be a template literals w/o expressions or string-only concatenation expressions or anything else.
+
+```
+export const myCollector = makeUsageCollector<Usage>({
+  type: 'string_literal_only',
+  ...
+});
+```
+
+### Usage
+
+```bash
+node scripts/telemetry_extract.js
+```
+
+This command has no additional flags or arguments. The `.telemetryrc.json` files specify the path to the directory where searching should start, output json files, and files to exclude.
+
+
+### Output
+
+
+The generated JSON files contain an ES mapping for each schema. This mapping is used to verify changes in the collectors and as the basis to map those fields into the external telemetry cluster.
+
+**Example**:
+
+```json
+{
+  "properties": {
+    "cloud": {
+      "properties": {
+        "isCloudEnabled": {
+          "type": "boolean"
+        }
+      }
+    }
+  }
+}
+```
+
+## Schema validation tool
+
+### Description
+
+The tool performs a number of checks on all telemetry collectors and verifies the following:
+
+1. Verifies the collector structure, fields, and returned values are using the appropriate types.
+2. Verifies that the collector `fetch` function Type matches the specified `schema` in the collector.
+3. Verifies that the collector `schema` matches the stored json schema .
+
+### Notes
+
+We don't catch every possible misuse of the collectors, but only the most common and critical ones.
+
+What will not be caught by the validator:
+
+* Mistyped SavedObject/CallCluster return value. Since the hits returned from ES can be typed to anything without any checks. It is advised to add functional tests that grabs the schema json file and checks that the returned usage matches the types exactly. 
+
+* Fields in the schema that are never collected. If you are trying to report a field from ES but that value is never stored in ES, the check will not be able to detect if that field is ever collected in the first palce. It is advised to add unit/functional tests to check that all the fields are being reported as expected.
+
+The tool looks for `.telemetryrc.json` files in the root of the project and in the `x-pack` dir for its runtime configurations.
+
+Currently auto-fixer (`--fix`) can automatically fix the json files with the following errors:
+
+* incompatible schema - this error means that the collector schema was changed but the stored json schema file was not updated.
+
+* unused schemas - this error means that a collector was removed or its `type` renamed, the json schema file contains a schema that does not have a corrisponding collector.
+
+### Usage
+
+```bash
+node scripts/telemetry_check --fix
+```
+
+* `--path` specifies a collector path instead of checking all collectors specified in the `.telemetryrc.json` files. Accepts a `.ts` file. The file must be discoverable by at least one rc file.
+* `--fix` tells the tool to try to fix as many violations as possible. All errors that tool won't be able to fix will be reported.

packages/kbn-telemetry-tools/babel.config.js

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+module.exports = {
+  presets: ['@kbn/babel-preset/node_preset'],
+  ignore: ['**/*.test.ts', '**/__fixture__/**'],
+};