From e36d2ca6a8bb2f00fbc3864170020838cadd4535 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Wed, 9 Sep 2020 14:02:47 +0200 Subject: [PATCH] [7.9] [APM] Use observer.hostname (#76074) (#76857) Co-authored-by: Elastic Machine Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> Co-authored-by: Joel Griffith Co-authored-by: John Schulz Co-authored-by: Brandon Morelli --- .../__snapshots__/apm_telemetry.test.ts.snap | 56 ++-- .../elasticsearch_fieldnames.test.ts.snap | 12 +- x-pack/plugins/apm/common/apm_telemetry.ts | 14 +- .../apm/common/elasticsearch_fieldnames.ts | 2 +- .../collect_data_telemetry/tasks.test.ts | 196 ++++++----- .../collect_data_telemetry/tasks.ts | 313 ++++++++++++------ .../apm/server/lib/apm_telemetry/index.ts | 43 ++- .../apm/server/lib/apm_telemetry/types.ts | 5 + x-pack/plugins/apm/server/plugin.ts | 1 + .../apm/typings/elasticsearch/aggregations.ts | 11 +- 10 files changed, 427 insertions(+), 226 deletions(-) diff --git a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap index 708758f2c6e585..e9763082a3999f 100644 --- a/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/apm_telemetry.test.ts.snap @@ -536,61 +536,54 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the }, "transaction_count": { "type": "long" - } - } - }, - "no_observer_name": { - "properties": { - "expected_metric_document_count": { - "type": "long" }, - "transaction_count": { + "ratio": { "type": "long" } } }, - "no_rum": { + "no_observer_name": { "properties": { "expected_metric_document_count": { "type": "long" }, "transaction_count": { "type": "long" - } - } - }, - "no_rum_no_observer_name": { - "properties": { - "expected_metric_document_count": { - "type": "long" }, - "transaction_count": { + "ratio": { "type": "long" } } }, - "only_rum": { + "with_country": { "properties": { "expected_metric_document_count": { "type": "long" }, "transaction_count": { "type": "long" - } - } - }, - "only_rum_no_observer_name": { - "properties": { - "expected_metric_document_count": { - "type": "long" }, - "transaction_count": { + "ratio": { "type": "long" } } } } }, + "environments": { + "properties": { + "services_without_environment": { + "type": "long" + }, + "services_with_multiple_environments": { + "type": "long" + }, + "top_enviroments": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, "cloud": { "properties": { "availability_zone": { @@ -952,6 +945,17 @@ exports[`APM telemetry helpers getApmTelemetry generates a JSON object with the } } }, + "environments": { + "properties": { + "took": { + "properties": { + "ms": { + "type": "long" + } + } + } + } + }, "groupings": { "properties": { "took": { diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index 7c42fb6f12a54a..d9e9fe4a827d7a 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -68,9 +68,9 @@ exports[`Error METRIC_SYSTEM_FREE_MEMORY 1`] = `undefined`; exports[`Error METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; -exports[`Error OBSERVER_LISTENING 1`] = `undefined`; +exports[`Error OBSERVER_HOSTNAME 1`] = `undefined`; -exports[`Error OBSERVER_NAME 1`] = `"an observer"`; +exports[`Error OBSERVER_LISTENING 1`] = `undefined`; exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`; @@ -216,9 +216,9 @@ exports[`Span METRIC_SYSTEM_FREE_MEMORY 1`] = `undefined`; exports[`Span METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; -exports[`Span OBSERVER_LISTENING 1`] = `undefined`; +exports[`Span OBSERVER_HOSTNAME 1`] = `undefined`; -exports[`Span OBSERVER_NAME 1`] = `"an observer"`; +exports[`Span OBSERVER_LISTENING 1`] = `undefined`; exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`; @@ -364,9 +364,9 @@ exports[`Transaction METRIC_SYSTEM_FREE_MEMORY 1`] = `undefined`; exports[`Transaction METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; -exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; +exports[`Transaction OBSERVER_HOSTNAME 1`] = `undefined`; -exports[`Transaction OBSERVER_NAME 1`] = `"an observer"`; +exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`; diff --git a/x-pack/plugins/apm/common/apm_telemetry.ts b/x-pack/plugins/apm/common/apm_telemetry.ts index 318b956cd3b3e9..3e885f4948c1e4 100644 --- a/x-pack/plugins/apm/common/apm_telemetry.ts +++ b/x-pack/plugins/apm/common/apm_telemetry.ts @@ -78,6 +78,7 @@ export function getApmTelemetryMapping() { properties: { expected_metric_document_count: long, transaction_count: long, + ratio: long, }, }; @@ -102,10 +103,14 @@ export function getApmTelemetryMapping() { properties: { current_implementation: aggregatedTransactionsProperties, no_observer_name: aggregatedTransactionsProperties, - no_rum: aggregatedTransactionsProperties, - no_rum_no_observer_name: aggregatedTransactionsProperties, - only_rum: aggregatedTransactionsProperties, - only_rum_no_observer_name: aggregatedTransactionsProperties, + with_country: aggregatedTransactionsProperties, + }, + }, + environments: { + properties: { + services_without_environment: long, + services_with_multiple_environments: long, + top_enviroments: keyword, }, }, cloud: { @@ -227,6 +232,7 @@ export function getApmTelemetryMapping() { agents: tookProperties, cardinality: tookProperties, cloud: tookProperties, + environments: tookProperties, groupings: tookProperties, indices_stats: tookProperties, integrations: tookProperties, diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts index 610a32e8e9b99b..084fdabe1e2974 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.ts @@ -31,7 +31,7 @@ export const USER_AGENT_NAME = 'user_agent.name'; export const DESTINATION_ADDRESS = 'destination.address'; -export const OBSERVER_NAME = 'observer.name'; +export const OBSERVER_HOSTNAME = 'observer.hostname'; export const OBSERVER_VERSION_MAJOR = 'observer.version_major'; export const OBSERVER_LISTENING = 'observer.listening'; export const PROCESSOR_EVENT = 'processor.event'; diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts index eafd0f04b9d10f..3f9e6ab79c3222 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.test.ts @@ -4,9 +4,12 @@ * you may not use this file except in compliance with the Elastic License. */ -import { AGENT_NAME } from '../../../../common/elasticsearch_fieldnames'; import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; import { tasks } from './tasks'; +import { + SERVICE_NAME, + SERVICE_ENVIRONMENT, +} from '../../../../common/elasticsearch_fieldnames'; describe('data telemetry collection tasks', () => { const indices = { @@ -16,103 +19,136 @@ describe('data telemetry collection tasks', () => { 'apm_oss.transactionIndices': 'apm-8.0.0-transaction', } as ApmIndicesConfig; + describe('environments', () => { + const task = tasks.find((t) => t.name === 'environments'); + + it('returns environment information', async () => { + const search = jest.fn().mockResolvedValueOnce({ + aggregations: { + environments: { + buckets: [ + { + key: 'production', + }, + { + key: 'testing', + }, + ], + }, + service_environments: { + buckets: [ + { + key: { + [SERVICE_NAME]: 'opbeans-node', + [SERVICE_ENVIRONMENT]: 'production', + }, + }, + { + key: { + [SERVICE_NAME]: 'opbeans-node', + [SERVICE_ENVIRONMENT]: null, + }, + }, + { + key: { + [SERVICE_NAME]: 'opbeans-java', + [SERVICE_ENVIRONMENT]: 'production', + }, + }, + { + key: { + [SERVICE_NAME]: 'opbeans-rum', + [SERVICE_ENVIRONMENT]: null, + }, + }, + ], + }, + }, + }); + + expect(await task?.executor({ search, indices } as any)).toEqual({ + environments: { + services_with_multiple_environments: 1, + services_without_environment: 2, + top_environments: ['production', 'testing'], + }, + }); + }); + }); + describe('aggregated_transactions', () => { const task = tasks.find((t) => t.name === 'aggregated_transactions'); - it('returns aggregated transaction counts', async () => { - // This mock implementation returns different values based on the parameters, - // which should simulate all the queries that are done. For most of them we'll - // simulate the number of buckets by using the length of the key, but for a - // couple we'll simulate being paginated by returning an after_key. - const search = jest.fn().mockImplementation((params) => { - const isRumResult = - params.body.query.bool.filter && - params.body.query.bool.filter.some( - (filter: any) => - filter.terms && filter.terms[AGENT_NAME]?.includes('rum-js') - ); - const isNonRumResult = - params.body.query.bool.filter && - params.body.query.bool.filter.some( - (filter: any) => - filter.terms && !filter.terms[AGENT_NAME]?.includes('rum-js') - ); - const isPagedResult = - !!params.body.aggs?.current_implementation?.composite.after || - !!params.body.aggs?.no_observer_name?.composite.after; - const isTotalResult = 'track_total_hits' in params.body; - const key = Object.keys(params.body.aggs ?? [])[0]; - - if (isRumResult) { - if (isTotalResult) { - return Promise.resolve({ hits: { total: { value: 3000 } } }); - } - } - - if (isNonRumResult) { - if (isTotalResult) { - return Promise.resolve({ hits: { total: { value: 2000 } } }); - } - } + describe('without transactions', () => { + it('returns an empty result', async () => { + const search = jest.fn().mockReturnValueOnce({ + hits: { + hits: [], + total: { + value: 0, + }, + }, + }); - if (isPagedResult && key) { - return Promise.resolve({ - hits: { total: { value: key.length } }, - aggregations: { [key]: { buckets: [{}] } }, - }); - } + expect(await task?.executor({ indices, search } as any)).toEqual({}); + }); + }); - if (isTotalResult) { - return Promise.resolve({ hits: { total: { value: 1000 } } }); - } + it('returns aggregated transaction counts', async () => { + const search = jest + .fn() + // The first call to `search` asks for a transaction to get + // a fixed date range. + .mockReturnValueOnce({ + hits: { + hits: [{ _source: { '@timestamp': new Date().toISOString() } }], + }, + total: { + value: 1, + }, + }) + // Later calls are all composite aggregations. We return 2 pages of + // results to test if scrolling works. + .mockImplementation((params) => { + let arrayLength = 1000; + let nextAfter: Record = { after_key: {} }; + + if (params.body.aggs.transaction_metric_groups.composite.after) { + arrayLength = 250; + nextAfter = {}; + } - if ( - key === 'current_implementation' || - (key === 'no_observer_name' && !isPagedResult) - ) { return Promise.resolve({ - hits: { total: { value: key.length } }, - aggregations: { - [key]: { after_key: {}, buckets: key.split('').map((_) => ({})) }, + hits: { + total: { + value: 5000, + }, }, - }); - } - - if (key) { - return Promise.resolve({ - hits: { total: { value: key.length } }, aggregations: { - [key]: { buckets: key.split('').map((_) => ({})) }, + transaction_metric_groups: { + buckets: new Array(arrayLength), + ...nextAfter, + }, }, }); - } - }); + }); expect(await task?.executor({ indices, search } as any)).toEqual({ aggregated_transactions: { current_implementation: { - expected_metric_document_count: 23, - transaction_count: 1000, + expected_metric_document_count: 1250, + transaction_count: 5000, + ratio: 0.25, }, no_observer_name: { - expected_metric_document_count: 17, - transaction_count: 1000, - }, - no_rum: { - expected_metric_document_count: 6, - transaction_count: 2000, - }, - no_rum_no_observer_name: { - expected_metric_document_count: 23, - transaction_count: 2000, - }, - only_rum: { - expected_metric_document_count: 8, - transaction_count: 3000, + expected_metric_document_count: 1250, + transaction_count: 5000, + ratio: 0.25, }, - only_rum_no_observer_name: { - expected_metric_document_count: 25, - transaction_count: 3000, + with_country: { + expected_metric_document_count: 1250, + transaction_count: 5000, + ratio: 0.25, }, }, }); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts index 840f47b0434184..04040d116bf2b8 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -3,7 +3,10 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { flatten, merge, sortBy, sum } from 'lodash'; +import { ValuesType } from 'utility-types'; +import { flatten, merge, sortBy, sum, pickBy } from 'lodash'; +import { AggregationOptionsByType } from '../../../../typings/elasticsearch/aggregations'; +import { ProcessorEvent } from '../../../../common/processor_event'; import { TelemetryTask } from '.'; import { AGENT_NAMES, RUM_AGENTS } from '../../../../common/agent_name'; import { @@ -16,7 +19,7 @@ import { CONTAINER_ID, ERROR_GROUP_ID, HOST_NAME, - OBSERVER_NAME, + OBSERVER_HOSTNAME, PARENT_ID, POD_NAME, PROCESSOR_EVENT, @@ -32,10 +35,8 @@ import { TRANSACTION_NAME, TRANSACTION_RESULT, TRANSACTION_TYPE, - USER_AGENT_NAME, USER_AGENT_ORIGINAL, } from '../../../../common/elasticsearch_fieldnames'; -import { ESFilter } from '../../../../typings/elasticsearch'; import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; import { Span } from '../../../../typings/es_schemas/ui/span'; @@ -57,79 +58,116 @@ export const tasks: TelemetryTask[] = [ // the transaction count for that time range. executor: async ({ indices, search }) => { async function getBucketCountFromPaginatedQuery( - key: string, - filter: ESFilter[], - count: number = 0, + sources: Array< + ValuesType[string] + >, + prevResult?: { + transaction_count: number; + expected_metric_document_count: number; + }, after?: any - ) { + ): Promise<{ + transaction_count: number; + expected_metric_document_count: number; + ratio: number; + }> { + // eslint-disable-next-line @typescript-eslint/naming-convention + let { expected_metric_document_count } = prevResult ?? { + transaction_count: 0, + expected_metric_document_count: 0, + }; + const params = { index: [indices['apm_oss.transactionIndices']], body: { size: 0, timeout, - query: { bool: { filter } }, + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + { range: { '@timestamp': { gte: start, lt: end } } }, + ], + }, + }, + track_total_hits: true, aggs: { - [key]: { + transaction_metric_groups: { composite: { ...(after ? { after } : {}), size: 10000, - sources: fieldMap[key].map((field) => ({ - [field]: { terms: { field, missing_bucket: true } }, - })), + sources: sources.map((source, index) => { + return { + [index]: source, + }; + }), }, }, }, }, }; + const result = await search(params); + let nextAfter: any; if (result.aggregations) { - nextAfter = result.aggregations[key].after_key; - count += result.aggregations[key].buckets.length; + nextAfter = result.aggregations.transaction_metric_groups.after_key; + // eslint-disable-next-line @typescript-eslint/camelcase + expected_metric_document_count += + result.aggregations.transaction_metric_groups.buckets.length; } if (nextAfter) { - count = await getBucketCountFromPaginatedQuery( - key, - filter, - count, + return await getBucketCountFromPaginatedQuery( + sources, + { + expected_metric_document_count, + transaction_count: result.hits.total.value, + }, nextAfter ); } - return count; + return { + expected_metric_document_count, + transaction_count: result.hits.total.value, + // eslint-disable-next-line @typescript-eslint/camelcase + ratio: expected_metric_document_count / result.hits.total.value, + }; } - async function totalSearch(filter: ESFilter[]) { - const result = await search({ - index: [indices['apm_oss.transactionIndices']], + // fixed date range for reliable results + const lastTransaction = ( + await search({ + index: indices['apm_oss.transactionIndices'], body: { - size: 0, - timeout, - query: { bool: { filter } }, - track_total_hits: true, + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + ], + }, + }, + size: 1, + sort: { + '@timestamp': 'desc', + }, }, - }); + }) + ).hits.hits[0] as { _source: { '@timestamp': string } }; - return result.hits.total.value; + if (!lastTransaction) { + return {}; } - const nonRumAgentNames = AGENT_NAMES.filter( - (name) => !RUM_AGENTS.includes(name) - ); + const end = + new Date(lastTransaction._source['@timestamp']).getTime() - + 5 * 60 * 1000; - const filter: ESFilter[] = [ - { term: { [PROCESSOR_EVENT]: 'transaction' } }, - { range: { '@timestamp': { gte: 'now-1m' } } }, - ]; - const noRumFilter = [ - ...filter, - { terms: { [AGENT_NAME]: nonRumAgentNames } }, - ]; - const rumFilter = [...filter, { terms: { [AGENT_NAME]: RUM_AGENTS } }]; + const start = end - 60 * 1000; - const baseFields = [ + const simpleTermFields = [ TRANSACTION_NAME, TRANSACTION_RESULT, TRANSACTION_TYPE, @@ -139,73 +177,61 @@ export const tasks: TelemetryTask[] = [ HOST_NAME, CONTAINER_ID, POD_NAME, - ]; - - const fieldMap: Record = { - current_implementation: [OBSERVER_NAME, ...baseFields, USER_AGENT_NAME], - no_observer_name: [...baseFields, USER_AGENT_NAME], - no_rum: [OBSERVER_NAME, ...baseFields], - no_rum_no_observer_name: baseFields, - only_rum: [OBSERVER_NAME, ...baseFields, USER_AGENT_NAME], - only_rum_no_observer_name: [...baseFields, USER_AGENT_NAME], - }; + ].map((field) => ({ terms: { field, missing_bucket: true } })); - // It would be more performant to do these in parallel, but we have different filters and keys and it's easier to - // understand if we make the code slower and longer - const countMap: Record = { - current_implementation: await getBucketCountFromPaginatedQuery( - 'current_implementation', - filter - ), - no_observer_name: await getBucketCountFromPaginatedQuery( - 'no_observer_name', - filter - ), - no_rum: await getBucketCountFromPaginatedQuery('no_rum', noRumFilter), - no_rum_no_observer_name: await getBucketCountFromPaginatedQuery( - 'no_rum_no_observer_name', - noRumFilter - ), - only_rum: await getBucketCountFromPaginatedQuery('only_rum', rumFilter), - only_rum_no_observer_name: await getBucketCountFromPaginatedQuery( - 'only_rum_no_observer_name', - rumFilter - ), + const observerHostname = { + terms: { field: OBSERVER_HOSTNAME, missing_bucket: true }, }; - const [allCount, noRumCount, rumCount] = await Promise.all([ - totalSearch(filter), - totalSearch(noRumFilter), - totalSearch(rumFilter), - ]); + const baseFields = [ + ...simpleTermFields, + // user_agent.name only for page-load transactions + { + terms: { + script: ` + if (doc['transaction.type'].value == 'page-load' && doc['user_agent.name'].size() > 0) { + return doc['user_agent.name'].value; + } - return { - aggregated_transactions: { - current_implementation: { - transaction_count: allCount, - expected_metric_document_count: countMap.current_implementation, - }, - no_observer_name: { - transaction_count: allCount, - expected_metric_document_count: countMap.no_observer_name, - }, - no_rum: { - transaction_count: noRumCount, - expected_metric_document_count: countMap.no_rum, + return null; + `, + missing_bucket: true, }, - no_rum_no_observer_name: { - transaction_count: noRumCount, - expected_metric_document_count: countMap.no_rum_no_observer_name, - }, - only_rum: { - transaction_count: rumCount, - expected_metric_document_count: countMap.only_rum, - }, - only_rum_no_observer_name: { - transaction_count: rumCount, - expected_metric_document_count: countMap.only_rum_no_observer_name, + }, + // transaction.root + { + terms: { + script: `return doc['parent.id'].size() == 0`, + missing_bucket: true, }, }, + ]; + + const results = { + current_implementation: await getBucketCountFromPaginatedQuery([ + ...baseFields, + observerHostname, + ]), + with_country: await getBucketCountFromPaginatedQuery([ + ...baseFields, + observerHostname, + { + terms: { + script: ` + if (doc['transaction.type'].value == 'page-load' && doc['client.geo.country_iso_code'].size() > 0) { + return doc['client.geo.country_iso_code'].value; + } + return null; + `, + missing_bucket: true, + }, + }, + ]), + no_observer_name: await getBucketCountFromPaginatedQuery(baseFields), + }; + + return { + aggregated_transactions: results, }; }, }, @@ -270,6 +296,89 @@ export const tasks: TelemetryTask[] = [ return { cloud }; }, }, + { + name: 'environments', + executor: async ({ indices, search }) => { + const response = await search({ + index: [indices['apm_oss.transactionIndices']], + body: { + query: { + bool: { + filter: [{ range: { '@timestamp': { gte: 'now-1d' } } }], + }, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + size: 5, + }, + }, + service_environments: { + composite: { + size: 1000, + sources: [ + { + [SERVICE_ENVIRONMENT]: { + terms: { + field: SERVICE_ENVIRONMENT, + missing_bucket: true, + }, + }, + }, + { + [SERVICE_NAME]: { + terms: { + field: SERVICE_NAME, + }, + }, + }, + ], + }, + }, + }, + }, + }); + + const topEnvironments = + response.aggregations?.environments.buckets.map( + (bucket) => bucket.key + ) ?? []; + const serviceEnvironments: Record> = {}; + + const buckets = response.aggregations?.service_environments.buckets ?? []; + + buckets.forEach((bucket) => { + const serviceName = bucket.key['service.name']; + const environment = (bucket.key['service.environment'] as unknown) as + | string + | null; + + const environments = serviceEnvironments[serviceName] ?? []; + + serviceEnvironments[serviceName] = environments.concat(environment); + }); + + const servicesWithoutEnvironment = Object.keys( + pickBy(serviceEnvironments, (environments) => + environments.includes(null) + ) + ); + + const servicesWithMultipleEnvironments = Object.keys( + pickBy(serviceEnvironments, (environments) => environments.length > 1) + ); + + return { + environments: { + services_without_environment: servicesWithoutEnvironment.length, + services_with_multiple_environments: + servicesWithMultipleEnvironments.length, + top_environments: topEnvironments as string[], + }, + }; + }, + }, { name: 'processor_events', executor: async ({ indices, search }) => { diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts index 2836cf100a4324..7b3cb6f2dc0cb3 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -5,8 +5,13 @@ */ import { Observable } from 'rxjs'; import { take } from 'rxjs/operators'; -import { CoreSetup, Logger } from 'src/core/server'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { DeepRequired } from 'utility-types'; +import { + CoreSetup, + Logger, + SavedObjectsErrorHelpers, +} from '../../../../../../src/core/server'; import { APMConfig } from '../..'; import { TaskManagerSetupContract, @@ -23,6 +28,7 @@ import { collectDataTelemetry, CollectTelemetryParams, } from './collect_data_telemetry'; +import { APMDataTelemetry } from './types'; const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task'; @@ -32,12 +38,14 @@ export async function createApmTelemetry({ usageCollector, taskManager, logger, + kibanaVersion, }: { core: CoreSetup; config$: Observable; usageCollector: UsageCollectionSetup; taskManager: TaskManagerSetupContract; logger: Logger; + kibanaVersion: string; }) { taskManager.registerTaskDefinitions({ [APM_TELEMETRY_TASK_NAME]: { @@ -91,7 +99,10 @@ export async function createApmTelemetry({ await savedObjectsClient.create( APM_TELEMETRY_SAVED_OBJECT_TYPE, - dataTelemetry, + { + ...dataTelemetry, + kibanaVersion, + }, { id: APM_TELEMETRY_SAVED_OBJECT_TYPE, overwrite: true } ); }; @@ -101,12 +112,14 @@ export async function createApmTelemetry({ schema: getApmTelemetryMapping(), fetch: async () => { try { - const data = ( + const { kibanaVersion: storedKibanaVersion, ...data } = ( await savedObjectsClient.get( APM_TELEMETRY_SAVED_OBJECT_TYPE, APM_TELEMETRY_SAVED_OBJECT_ID ) - ).attributes; + ).attributes as { kibanaVersion: string } & DeepRequired< + APMDataTelemetry + >; return data; } catch (err) { @@ -122,7 +135,7 @@ export async function createApmTelemetry({ usageCollector.registerCollector(collector); - core.getStartServices().then(([_coreStart, pluginsStart]) => { + core.getStartServices().then(async ([_coreStart, pluginsStart]) => { const { taskManager: taskManagerStart } = pluginsStart as { taskManager: TaskManagerStartContract; }; @@ -137,5 +150,25 @@ export async function createApmTelemetry({ params: {}, state: {}, }); + + try { + const currentData = ( + await savedObjectsClient.get( + APM_TELEMETRY_SAVED_OBJECT_TYPE, + APM_TELEMETRY_SAVED_OBJECT_ID + ) + ).attributes as { kibanaVersion?: string }; + + if (currentData.kibanaVersion !== kibanaVersion) { + logger.debug( + `Stored telemetry is out of date. Task will run immediately. Stored: ${currentData.kibanaVersion}, expected: ${kibanaVersion}` + ); + taskManagerStart.runNow(APM_TELEMETRY_TASK_NAME); + } + } catch (err) { + if (!SavedObjectsErrorHelpers.isNotFoundError(err)) { + logger.warn('Failed to fetch saved telemetry data.'); + } + } }); } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts index 82e4d1e395ed34..c7af292e817c7a 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -30,6 +30,11 @@ export type APMDataTelemetry = DeepPartial<{ patch: number; }; }; + environments: { + services_without_environments: number; + services_with_multiple_environments: number; + top_environments: string[]; + }; aggregated_transactions: { current_implementation: AggregatedTransactionsCounts; no_observer_name: AggregatedTransactionsCounts; diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index deafda67b806d7..5efed7b856a03a 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -100,6 +100,7 @@ export class APMPlugin implements Plugin { usageCollector: plugins.usageCollection, taskManager: plugins.taskManager, logger: this.logger, + kibanaVersion: this.initContext.env.packageInfo.version, }); } diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index ac7499c23e9268..bd9a90f8f8d9db 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -51,7 +51,12 @@ type GetCompositeKeys< type CompositeOptionsSource = Record< string, - { terms: { field: string; missing_bucket?: boolean } } | undefined + | { + terms: ({ field: string } | { script: Script }) & { + missing_bucket?: boolean; + }; + } + | undefined >; export interface AggregationOptionsByType { @@ -288,7 +293,9 @@ interface AggregationResponsePart< } | undefined; composite: { - after_key: Record, number>; + after_key: { + [key in GetCompositeKeys]: TAggregationOptionsMap; + }; buckets: Array< { key: Record, number>;