Add ML usage collector descriptions to the schema. (#100387) (#100860)

* Add ML usage collector descriptions to the schema. * Update the telemetry mappings file
elastic · May 28, 2021 · 9f19a64 · 9f19a64
1 parent 9d485b5
commit 9f19a64
Show file tree

Hide file tree

Showing 2 changed files with 274 additions and 58 deletions.
diff --git a/x-pack/plugins/security_solution/server/usage/collector.ts b/x-pack/plugins/security_solution/server/usage/collector.ts
@@ -250,43 +250,172 @@ export const registerCollector: RegisterCollector = ({
         ml_jobs: {
           type: 'array',
           items: {
-            job_id: { type: 'keyword' },
-            open_time: { type: 'keyword' },
-            create_time: { type: 'keyword' },
-            finished_time: { type: 'keyword' },
-            state: { type: 'keyword' },
+            job_id: {
+              type: 'keyword',
+              _meta: { description: 'Identifier for the anomaly detection job' },
+            },
+            open_time: {
+              type: 'keyword',
+              _meta: {
+                description: 'For open jobs only, the elapsed time for which the job has been open',
+              },
+            },
+            create_time: {
+              type: 'keyword',
+              _meta: { description: 'The time the job was created' },
+            },
+            finished_time: {
+              type: 'keyword',
+              _meta: {
+                description: 'If the job closed or failed, this is the time the job finished',
+              },
+            },
+            state: {
+              type: 'keyword',
+              _meta: { description: 'The status of the anomaly detection job' },
+            },
             data_counts: {
-              bucket_count: { type: 'long' },
-              empty_bucket_count: { type: 'long' },
-              input_bytes: { type: 'long' },
-              input_record_count: { type: 'long' },
-              last_data_time: { type: 'long' },
-              processed_record_count: { type: 'long' },
+              bucket_count: {
+                type: 'long',
+                _meta: { description: 'The number of buckets processed' },
+              },
+              empty_bucket_count: {
+                type: 'long',
+                _meta: { description: 'The number of buckets which did not contain any data' },
+              },
+              input_bytes: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'The number of bytes of input data posted to the anomaly detection job',
+                },
+              },
+              input_record_count: {
+                type: 'long',
+                _meta: {
+                  description: 'The number of input documents posted to the anomaly detection job',
+                },
+              },
+              last_data_time: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'The timestamp at which data was last analyzed, according to server time',
+                },
+              },
+              processed_record_count: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'The number of input documents that have been processed by the anomaly detection job',
+                },
+              },
             },
             model_size_stats: {
-              bucket_allocation_failures_count: { type: 'long' },
-              model_bytes: { type: 'long' },
-              model_bytes_exceeded: { type: 'long' },
-              model_bytes_memory_limit: { type: 'long' },
-              peak_model_bytes: { type: 'long' },
+              bucket_allocation_failures_count: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'The number of buckets for which new entities in incoming data were not processed due to insufficient model memory',
+                },
+              },
+              model_bytes: {
+                type: 'long',
+                _meta: { description: 'The number of bytes of memory used by the models' },
+              },
+              model_bytes_exceeded: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'The number of bytes over the high limit for memory usage at the last allocation failure',
+                },
+              },
+              model_bytes_memory_limit: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'The upper limit for model memory usage, checked on increasing values',
+                },
+              },
+              peak_model_bytes: {
+                type: 'long',
+                _meta: {
+                  description: 'The peak number of bytes of memory ever used by the models',
+                },
+              },
             },
             timing_stats: {
-              bucket_count: { type: 'long' },
-              exponential_average_bucket_processing_time_ms: { type: 'long' },
-              exponential_average_bucket_processing_time_per_hour_ms: { type: 'long' },
-              maximum_bucket_processing_time_ms: { type: 'long' },
-              minimum_bucket_processing_time_ms: { type: 'long' },
-              total_bucket_processing_time_ms: { type: 'long' },
+              bucket_count: {
+                type: 'long',
+                _meta: { description: 'The number of buckets processed' },
+              },
+              exponential_average_bucket_processing_time_ms: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'Exponential moving average of all bucket processing times, in milliseconds',
+                },
+              },
+              exponential_average_bucket_processing_time_per_hour_ms: {
+                type: 'long',
+                _meta: {
+                  description:
+                    'Exponentially-weighted moving average of bucket processing times calculated in a 1 hour time window, in milliseconds',
+                },
+              },
+              maximum_bucket_processing_time_ms: {
+                type: 'long',
+                _meta: {
+                  description: 'Maximum among all bucket processing times, in milliseconds',
+                },
+              },
+              minimum_bucket_processing_time_ms: {
+                type: 'long',
+                _meta: {
+                  description: 'Minimum among all bucket processing times, in milliseconds',
+                },
+              },
+              total_bucket_processing_time_ms: {
+                type: 'long',
+                _meta: { description: 'Sum of all bucket processing times, in milliseconds' },
+              },
             },
             datafeed: {
-              datafeed_id: { type: 'keyword' },
-              state: { type: 'keyword' },
+              datafeed_id: {
+                type: 'keyword',
+                _meta: {
+                  description: 'A numerical character string that uniquely identifies the datafeed',
+                },
+              },
+              state: {
+                type: 'keyword',
+                _meta: { description: 'The status of the datafeed' },
+              },
               timing_stats: {
-                average_search_time_per_bucket_ms: { type: 'long' },
-                bucket_count: { type: 'long' },
-                exponential_average_search_time_per_hour_ms: { type: 'long' },
-                search_count: { type: 'long' },
-                total_search_time_ms: { type: 'long' },
+                average_search_time_per_bucket_ms: {
+                  type: 'long',
+                  _meta: { description: 'The average search time per bucket, in milliseconds' },
+                },
+                bucket_count: {
+                  type: 'long',
+                  _meta: { description: 'The number of buckets processed' },
+                },
+                exponential_average_search_time_per_hour_ms: {
+                  type: 'long',
+                  _meta: {
+                    description: 'The exponential average search time per hour, in milliseconds',
+                  },
+                },
+                search_count: {
+                  type: 'long',
+                  _meta: { description: 'The number of searches run by the datafeed' },
+                },
+                total_search_time_ms: {
+                  type: 'long',
+                  _meta: {
+                    description: 'The total time the datafeed spent searching, in milliseconds',
+                  },
+                },
               },
             },
           },