From 7bbe02825a24dc396b20f5dd6b08c1b3ca5be25e Mon Sep 17 00:00:00 2001 From: Dima Arnautov Date: Fri, 10 Jan 2020 23:19:26 +0100 Subject: [PATCH] [ML] APM modules configs for RUM Javascript and NodeJS (#53792) * [ML] apm modules * [ML] apm modules * [ML] update mocha test * [ML] fix config * [ML] single line JSON formatting for queries * [ML] remove an empty path component with a trailing slash * [ML] change detector descriptions, remove scroll size * [ML] remove chunking_config from datafeeds * [ML] fix configs Co-authored-by: Elastic Machine --- .../application/util/custom_url_utils.test.ts | 92 +++++++++++++++++++ .../application/util/custom_url_utils.ts | 54 +++++++---- .../__tests__/data_recognizer.js | 2 + .../modules/apm_jsbase/logo.json | 3 + .../modules/apm_jsbase/manifest.json | 53 +++++++++++ .../ml/abnormal_span_durations_jsbase.json | 41 +++++++++ ...ous_error_rate_for_user_agents_jsbase.json | 40 ++++++++ ...tafeed_abnormal_span_durations_jsbase.json | 15 +++ ...ous_error_rate_for_user_agents_jsbase.json | 15 +++ .../datafeed_decreased_throughput_jsbase.json | 27 ++++++ ...afeed_high_count_by_user_agent_jsbase.json | 16 ++++ .../ml/decreased_throughput_jsbase.json | 37 ++++++++ .../ml/high_count_by_user_agent_jsbase.json | 38 ++++++++ .../modules/apm_nodejs/logo.json | 3 + .../modules/apm_nodejs/manifest.json | 42 +++++++++ .../ml/abnormal_span_durations_nodejs.json | 41 +++++++++ .../ml/abnormal_trace_durations_nodejs.json | 40 ++++++++ ...tafeed_abnormal_span_durations_nodejs.json | 15 +++ ...afeed_abnormal_trace_durations_nodejs.json | 13 +++ .../datafeed_decreased_throughput_nodejs.json | 27 ++++++ .../ml/decreased_throughput_nodejs.json | 38 ++++++++ 21 files changed, 633 insertions(+), 19 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/logo.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/manifest.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/abnormal_span_durations_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/anomalous_error_rate_for_user_agents_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_abnormal_span_durations_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_anomalous_error_rate_for_user_agents_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_decreased_throughput_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_high_count_by_user_agent_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/decreased_throughput_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/high_count_by_user_agent_jsbase.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/logo.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/manifest.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_span_durations_nodejs.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_trace_durations_nodejs.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_span_durations_nodejs.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_trace_durations_nodejs.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_decreased_throughput_nodejs.json create mode 100644 x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/decreased_throughput_nodejs.json diff --git a/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.test.ts b/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.test.ts index a7afee237dba9c7..8cdaa192fcbc9b0 100644 --- a/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.test.ts +++ b/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.test.ts @@ -345,6 +345,98 @@ describe('ML - custom URL utils', () => { ); }); + test('returns expected URL for APM', () => { + const urlConfig = { + url_name: 'APM', + time_range: '2h', + url_value: + 'apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:"$trace.id$" and transaction.name:"$transaction.name$"&_g=()', + }; + + const testRecords = { + job_id: 'abnormal_trace_durations_nodejs', + result_type: 'record', + probability: 0.025597710862701226, + multi_bucket_impact: 5, + record_score: 13.124152090331723, + initial_record_score: 13.124152090331723, + bucket_span: 900, + detector_index: 0, + is_interim: false, + timestamp: 1573339500000, + by_field_name: 'transaction.name', + by_field_value: 'GET /test-data', + function: 'high_mean', + function_description: 'mean', + typical: [802.0600710562369], + actual: [761.1531339031332], + field_name: 'transaction.duration.us', + influencers: [ + { + influencer_field_name: 'transaction.name', + influencer_field_values: ['GET /test-data'], + }, + { + influencer_field_name: 'trace.id', + influencer_field_values: [ + '000a09d58a428f38550e7e87637733c1', + '0039c771d8bbadf6137767d3aeb89f96', + '01279ed5bb9f4249e3822d16dec7f2f2', + ], + }, + { + influencer_field_name: 'service.name', + influencer_field_values: ['example-service'], + }, + ], + 'trace.id': [ + '000a09d58a428f38550e7e87637733c1', + '0039c771d8bbadf6137767d3aeb89f96', + '01279ed5bb9f4249e3822d16dec7f2f2', + ], + 'service.name': ['example-service'], + 'transaction.name': ['GET /test-data'], + earliest: '2019-11-09T20:45:00.000Z', + latest: '2019-11-10T01:00:00.000Z', + }; + + expect(getUrlForRecord(urlConfig, testRecords)).toBe( + 'apm#/traces?rangeFrom=2019-11-09T20:45:00.000Z&rangeTo=2019-11-10T01:00:00.000Z&kuery=(trace.id:"000a09d58a428f38550e7e87637733c1" OR trace.id:"0039c771d8bbadf6137767d3aeb89f96" OR trace.id:"01279ed5bb9f4249e3822d16dec7f2f2") AND transaction.name:"GET%20%2Ftest-data"&_g=()' + ); + }); + + test('removes an empty path component with a trailing slash', () => { + const urlConfig = { + url_name: 'APM', + time_range: '2h', + url_value: + 'apm#/services/$service.name$/transactions?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request', + }; + + const testRecords = { + job_id: 'decreased_throughput_jsbase', + result_type: 'record', + probability: 8.91350850732573e-9, + multi_bucket_impact: 5, + record_score: 93.63625728951217, + initial_record_score: 93.63625728951217, + bucket_span: 900, + detector_index: 0, + is_interim: false, + timestamp: 1573266600000, + function: 'low_count', + function_description: 'count', + typical: [100615.66506877479], + actual: [25251], + earliest: '2019-11-09T00:30:00.000Z', + latest: '2019-11-09T04:45:00.000Z', + }; + + expect(getUrlForRecord(urlConfig, testRecords)).toBe( + 'apm#/services/transactions?rangeFrom=2019-11-09T00:30:00.000Z&rangeTo=2019-11-09T04:45:00.000Z&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request' + ); + }); + test('returns expected URL for other type URL', () => { expect(getUrlForRecord(TEST_OTHER_URL, TEST_RECORD)).toBe( 'http://airlinecodes.info/airline-code-AAL' diff --git a/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.ts b/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.ts index e2f2dc0ad0fe848..7774f6dec0c9535 100644 --- a/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.ts +++ b/x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.ts @@ -97,7 +97,11 @@ export function openCustomUrlWindow(fullUrl: string, urlConfig: UrlConfig) { // a Kibana Discover or Dashboard page running on the same server as this ML plugin. function isKibanaUrl(urlConfig: UrlConfig) { const urlValue = urlConfig.url_value; - return urlValue.startsWith('kibana#/discover') || urlValue.startsWith('kibana#/dashboard'); + return ( + urlValue.startsWith('kibana#/discover') || + urlValue.startsWith('kibana#/dashboard') || + urlValue.startsWith('apm#/') + ); } /** @@ -136,13 +140,14 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) commonEscapeCallback ); - return str.replace(/\$([^?&$\'"]+)\$/g, (match, name: string) => { + // Looking for a $token$ with an optional trailing slash + return str.replace(/\$([^?&$\'"]+)\$(\/)?/g, (match, name: string, slash: string = '') => { // Use lodash get to allow nested JSON fields to be retrieved. let tokenValue: string | string[] | undefined = get(record, name); tokenValue = Array.isArray(tokenValue) ? tokenValue[0] : tokenValue; - // If property not found string is not replaced. - return tokenValue === undefined ? match : getResultTokenValue(tokenValue); + // If property not found token is replaced with an empty string. + return tokenValue === undefined ? '' : getResultTokenValue(tokenValue) + slash; }); }; @@ -155,7 +160,7 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) commonEscapeCallback ); return str.replace( - /(.+query:')([^']*)('.+)/, + /(.+query:'|.+&kuery=)([^']*)(['&].+)/, (fullMatch, prefix: string, queryString: string, postfix: string) => { const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues); @@ -170,28 +175,39 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc) const queryParts: string[] = []; const joinOperator = ' AND '; - for (let i = 0; i < queryFields.length; i++) { + fieldsLoop: for (let i = 0; i < queryFields.length; i++) { const field = queryFields[i]; // Use lodash get to allow nested JSON fields to be retrieved. - const tokenValues: string[] | string | null = get(record, field) || null; + let tokenValues: string[] | string | null = get(record, field) || null; if (tokenValues === null) { continue; } + tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues]; + // Create a pair `influencerField:value`. // In cases where there are multiple influencer field values for an anomaly // combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`. - let result = (Array.isArray(tokenValues) ? tokenValues : [tokenValues]) - .map(value => `${field}:"${getResultTokenValue(value)}"`) - .join(' OR '); - result = tokenValues.length > 1 ? `(${result})` : result; - - // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query. - availableCharactersLeft -= result.length - (i === 0 ? 0 : joinOperator.length); - - if (availableCharactersLeft <= 0) { - break; - } else { - queryParts.push(result); + let result = ''; + for (let j = 0; j < tokenValues.length; j++) { + const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue( + tokenValues[j] + )}"`; + + // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query. + if (availableCharactersLeft < part.length) { + if (result.length > 0) { + queryParts.push(j > 0 ? `(${result})` : result); + } + break fieldsLoop; + } + + result += part; + + availableCharactersLeft -= result.length; + } + + if (result.length > 0) { + queryParts.push(tokenValues.length > 1 ? `(${result})` : result); } } diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js b/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js index cb268ffede7faea..9c5048daeee3f0b 100644 --- a/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js @@ -12,6 +12,8 @@ describe('ML - data recognizer', () => { const moduleIds = [ 'apache_ecs', + 'apm_jsbase', + 'apm_nodejs', 'apm_transaction', 'auditbeat_process_docker_ecs', 'auditbeat_process_hosts_ecs', diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/logo.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/logo.json new file mode 100644 index 000000000000000..3905c809fbd7a8f --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "apmApp" +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/manifest.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/manifest.json new file mode 100644 index 000000000000000..e463b34be0fc27f --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/manifest.json @@ -0,0 +1,53 @@ +{ + "id": "apm_jsbase", + "title": "APM: RUM Javascript", + "description": "Detect problematic spans and identify user agents that are potentially causing issues.", + "type": "APM data", + "logoFile": "logo.json", + "defaultIndexPattern": "apm-*", + "query": { + "bool": { + "filter": [{ "term": { "agent.name": "js-base" } }] + } + }, + "jobs": [ + { + "id": "abnormal_span_durations_jsbase", + "file": "abnormal_span_durations_jsbase.json" + }, + { + "id": "anomalous_error_rate_for_user_agents_jsbase", + "file": "anomalous_error_rate_for_user_agents_jsbase.json" + }, + { + "id": "decreased_throughput_jsbase", + "file": "decreased_throughput_jsbase.json" + }, + { + "id": "high_count_by_user_agent_jsbase", + "file": "high_count_by_user_agent_jsbase.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-abnormal_span_durations_jsbase", + "file": "datafeed_abnormal_span_durations_jsbase.json", + "job_id": "abnormal_span_durations_jsbase" + }, + { + "id": "datafeed-anomalous_error_rate_for_user_agents_jsbase", + "file": "datafeed_anomalous_error_rate_for_user_agents_jsbase.json", + "job_id": "anomalous_error_rate_for_user_agents_jsbase" + }, + { + "id": "datafeed-decreased_throughput_jsbase", + "file": "datafeed_decreased_throughput_jsbase.json", + "job_id": "decreased_throughput_jsbase" + }, + { + "id": "datafeed-high_count_by_user_agent_jsbase", + "file": "datafeed_high_count_by_user_agent_jsbase.json", + "job_id": "high_count_by_user_agent_jsbase" + } + ] +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/abnormal_span_durations_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/abnormal_span_durations_jsbase.json new file mode 100644 index 000000000000000..e0b51a4dcd05e9c --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/abnormal_span_durations_jsbase.json @@ -0,0 +1,41 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "APM JSBase: Looks for spans that are taking longer than usual to process.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "increased span duration", + "function": "high_mean", + "field_name": "span.duration.us", + "partition_field_name": "span.type" + } + ], + "influencers": [ + "span.type", + "trace.id", + "span.name", + "service.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-apm-jsbase", + "custom_urls": [ + { + "url_name": "APM", + "time_range": "2h", + "url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\"&_g=()" + } + ] + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/anomalous_error_rate_for_user_agents_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/anomalous_error_rate_for_user_agents_jsbase.json new file mode 100644 index 000000000000000..66fd9858c6885a7 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/anomalous_error_rate_for_user_agents_jsbase.json @@ -0,0 +1,40 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "APM JSBase: Detects user agents that are encountering errors at an above normal rate. This can help detect browser compatibility issues.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high error rate for user agent", + "function": "high_non_zero_count", + "partition_field_name": "user_agent.name" + } + ], + "influencers": [ + "user_agent.name", + "error.exception.message.keyword", + "error.page.url", + "service.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-apm-jsbase", + "custom_urls": [ + { + "url_name": "APM", + "time_range": "2h", + "url_value": "apm#/services/$service.name$/errors?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=user_agent.name:\"$user_agent.name$\"&_g=()" + } + ] + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_abnormal_span_durations_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_abnormal_span_durations_jsbase.json new file mode 100644 index 000000000000000..7ecbe2890b826c3 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_abnormal_span_durations_jsbase.json @@ -0,0 +1,15 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { "bool": { "filter": { "term": { "agent.name": "js-base" } } } }, + { "bool": { "filter": { "term": { "processor.event": "span" } } } } + ] + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_anomalous_error_rate_for_user_agents_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_anomalous_error_rate_for_user_agents_jsbase.json new file mode 100644 index 000000000000000..fbfedcbf47561bd --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_anomalous_error_rate_for_user_agents_jsbase.json @@ -0,0 +1,15 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { "bool": { "filter": { "term": { "agent.name": "js-base" } } } }, + { "exists": { "field": "user_agent.name" } } + ] + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_decreased_throughput_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_decreased_throughput_jsbase.json new file mode 100644 index 000000000000000..48cba1f1578154b --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_decreased_throughput_jsbase.json @@ -0,0 +1,27 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": { "term": { "agent.name": "js-base" } } + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "900000ms" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_high_count_by_user_agent_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_high_count_by_user_agent_jsbase.json new file mode 100644 index 000000000000000..18ca6b138928717 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_high_count_by_user_agent_jsbase.json @@ -0,0 +1,16 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { "bool": { "filter": { "term": { "agent.name": "js-base" } } } }, + { "bool": { "filter": [{ "exists": { "field": "user_agent.name" } }] } }, + { "bool": { "filter": { "term": { "processor.event": "transaction" } } } } + ] + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/decreased_throughput_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/decreased_throughput_jsbase.json new file mode 100644 index 000000000000000..4bc8757f19dc967 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/decreased_throughput_jsbase.json @@ -0,0 +1,37 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "APM JSBase: Identifies periods during which the application is processing fewer requests than normal.", + "analysis_config": { + "summary_count_field_name": "doc_count", + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "low throughput", + "function": "low_count" + } + ], + "influencers": [ + "service.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-apm-jsbase", + "custom_urls": [ + { + "url_name": "APM", + "time_range": "2h", + "url_value": "apm#/services?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request" + } + ] + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/high_count_by_user_agent_jsbase.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/high_count_by_user_agent_jsbase.json new file mode 100644 index 000000000000000..7e1316359eabbd5 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/high_count_by_user_agent_jsbase.json @@ -0,0 +1,38 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "APM JSBase: Detects user agents that are making requests at a suspiciously high rate. This is useful in identifying bots.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "high request rate for user agent", + "function": "high_non_zero_count", + "partition_field_name": "user_agent.name" + } + ], + "influencers": [ + "user_agent.name", + "service.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "32mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-apm-jsbase", + "custom_urls": [ + { + "url_name": "APM", + "time_range": "2h", + "url_value": "apm#/services/$service.name$/transactions?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=user_agent.name:\"$user_agent.name$\"&_g=()" + } + ] + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/logo.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/logo.json new file mode 100644 index 000000000000000..3905c809fbd7a8f --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/logo.json @@ -0,0 +1,3 @@ +{ + "icon": "apmApp" +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/manifest.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/manifest.json new file mode 100644 index 000000000000000..1865a33a1d30114 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/manifest.json @@ -0,0 +1,42 @@ +{ + "id": "apm_nodejs", + "title": "APM: NodeJS", + "description": "Detect abnormal traces, anomalous spans, and identify periods of decreased throughput.", + "type": "APM data", + "logoFile": "logo.json", + "defaultIndexPattern": "apm-*", + "query": { + "bool": { "filter": [{ "term": { "agent.name": "nodejs" } }] } + }, + "jobs": [ + { + "id": "abnormal_span_durations_nodejs", + "file": "abnormal_span_durations_nodejs.json" + }, + { + "id": "abnormal_trace_durations_nodejs", + "file": "abnormal_trace_durations_nodejs.json" + }, + { + "id": "decreased_throughput_nodejs", + "file": "decreased_throughput_nodejs.json" + } + ], + "datafeeds": [ + { + "id": "datafeed-abnormal_span_durations_nodejs", + "file": "datafeed_abnormal_span_durations_nodejs.json", + "job_id": "abnormal_span_durations_nodejs" + }, + { + "id": "datafeed-abnormal_trace_durations_nodejs", + "file": "datafeed_abnormal_trace_durations_nodejs.json", + "job_id": "abnormal_trace_durations_nodejs" + }, + { + "id": "datafeed-decreased_throughput_nodejs", + "file": "datafeed_decreased_throughput_nodejs.json", + "job_id": "decreased_throughput_nodejs" + } + ] +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_span_durations_nodejs.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_span_durations_nodejs.json new file mode 100644 index 000000000000000..1a8318437790ef7 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_span_durations_nodejs.json @@ -0,0 +1,41 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "APM NodeJS: Looks for spans that are taking longer than usual to process.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "increased span duration", + "function": "high_mean", + "field_name": "span.duration.us", + "partition_field_name": "span.type" + } + ], + "influencers": [ + "span.type", + "trace.id", + "span.name", + "service.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "128mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-apm-nodejs", + "custom_urls": [ + { + "url_name": "APM", + "time_range": "2h", + "url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\"&_g=()" + } + ] + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_trace_durations_nodejs.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_trace_durations_nodejs.json new file mode 100644 index 000000000000000..875b49e895a0085 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/abnormal_trace_durations_nodejs.json @@ -0,0 +1,40 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "APM NodeJS: Identifies trace transactions that are processing more slowly than usual.", + "analysis_config": { + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "increased trace duration", + "function": "high_mean", + "field_name": "transaction.duration.us", + "by_field_name": "transaction.name" + } + ], + "influencers": [ + "transaction.name", + "trace.id", + "service.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "256mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-apm-nodejs", + "custom_urls": [ + { + "url_name": "APM", + "time_range": "2h", + "url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\" and transaction.name:\"$transaction.name$\"&_g=()" + } + ] + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_span_durations_nodejs.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_span_durations_nodejs.json new file mode 100644 index 000000000000000..3e4f4877bd04242 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_span_durations_nodejs.json @@ -0,0 +1,15 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must": [ + { "bool": { "filter": { "term": { "agent.name": "nodejs" } } } }, + { "bool": { "filter": { "term": { "processor.event": "span" } } } } + ] + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_trace_durations_nodejs.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_trace_durations_nodejs.json new file mode 100644 index 000000000000000..d87f809a4994062 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_abnormal_trace_durations_nodejs.json @@ -0,0 +1,13 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "must_not": [{ "exists": { "field": "parent.id" } }], + "must": [{ "bool": { "filter": { "term": { "agent.name": "nodejs" } } } }] + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_decreased_throughput_nodejs.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_decreased_throughput_nodejs.json new file mode 100644 index 000000000000000..451957c327dd04c --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/datafeed_decreased_throughput_nodejs.json @@ -0,0 +1,27 @@ +{ + "job_id": "JOB_ID", + "indices": [ + "INDEX_PATTERN_NAME" + ], + "max_empty_searches": 10, + "query": { + "bool": { + "filter": { "term": { "agent.name": "nodejs" } } + } + }, + "aggregations": { + "buckets": { + "date_histogram": { + "field": "@timestamp", + "fixed_interval": "900000ms" + }, + "aggregations": { + "@timestamp": { + "max": { + "field": "@timestamp" + } + } + } + } + } +} diff --git a/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/decreased_throughput_nodejs.json b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/decreased_throughput_nodejs.json new file mode 100644 index 000000000000000..f63c6289a5cd904 --- /dev/null +++ b/x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_nodejs/ml/decreased_throughput_nodejs.json @@ -0,0 +1,38 @@ +{ + "job_type": "anomaly_detector", + "groups": [ + "apm" + ], + "description": "APM NodeJS: Identifies periods during which the application is processing fewer requests than normal.", + "analysis_config": { + "summary_count_field_name": "doc_count", + "bucket_span": "15m", + "detectors": [ + { + "detector_description": "low throughput", + "function": "low_count" + } + ], + "influencers": [ + "transaction.name", + "service.name" + ] + }, + "allow_lazy_open": true, + "analysis_limits": { + "model_memory_limit": "10mb" + }, + "data_description": { + "time_field": "@timestamp" + }, + "custom_settings": { + "created_by": "ml-module-apm-nodejs", + "custom_urls": [ + { + "url_name": "APM", + "time_range": "2h", + "url_value": "apm#/services?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request" + } + ] + } +}