From 6795a4c100c384356ede67f72249e8e02bceadd9 Mon Sep 17 00:00:00 2001 From: "Christiane (Tina) Heiligers" Date: Tue, 30 Jun 2020 09:19:53 -0700 Subject: [PATCH] [Usage Collection] Report nodes feature usage (#70108) (#70329) * Adds nodes feature usage stats merged into cluster_stats.nodes when usage collection is local --- .../__tests__/get_local_stats.js | 65 +++++++++++++-- .../telemetry_collection/get_local_stats.ts | 14 +++- .../get_nodes_usage.test.ts | 80 ++++++++++++++++++ .../telemetry_collection/get_nodes_usage.ts | 81 +++++++++++++++++++ .../apis/telemetry/telemetry_local.js | 1 + .../get_stats_with_xpack.test.ts.snap | 44 +++++++++- .../get_stats_with_xpack.test.ts | 25 ++++++ 7 files changed, 299 insertions(+), 11 deletions(-) create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts create mode 100644 src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts diff --git a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js index 29076537e9ae88..e78b92498e6e78 100644 --- a/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js +++ b/src/plugins/telemetry/server/telemetry_collection/__tests__/get_local_stats.js @@ -19,11 +19,12 @@ import expect from '@kbn/expect'; import sinon from 'sinon'; +import { merge, omit } from 'lodash'; +import { TIMEOUT } from '../constants'; import { mockGetClusterInfo } from './get_cluster_info'; import { mockGetClusterStats } from './get_cluster_stats'; -import { omit } from 'lodash'; import { getLocalStats, handleLocalStats } from '../get_local_stats'; const mockUsageCollection = (kibanaUsage = {}) => ({ @@ -51,10 +52,26 @@ const getMockServer = (getCluster = sinon.stub()) => ({ elasticsearch: { getCluster }, }, }); +function mockGetNodesUsage(callCluster, nodesUsage, req) { + callCluster + .withArgs( + req, + { + method: 'GET', + path: '/_nodes/usage', + query: { + timeout: TIMEOUT, + }, + }, + 'transport.request' + ) + .returns(nodesUsage); +} -function mockGetLocalStats(callCluster, clusterInfo, clusterStats, req) { +function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) { mockGetClusterInfo(callCluster, clusterInfo, req); mockGetClusterStats(callCluster, clusterStats, req); + mockGetNodesUsage(callCluster, nodesUsage, req); } describe('get_local_stats', () => { @@ -68,6 +85,28 @@ describe('get_local_stats', () => { number: version, }, }; + const nodesUsage = [ + { + node_id: 'some_node_id', + timestamp: 1588617023177, + since: 1588616945163, + rest_actions: { + nodes_usage_action: 1, + create_index_action: 1, + document_get_action: 1, + search_action: 19, + nodes_info_action: 36, + }, + aggregations: { + terms: { + bytes: 2, + }, + scripted_metric: { + other: 7, + }, + }, + }, + ]; const clusterStats = { _nodes: { failed: 123 }, cluster_name: 'real-cool', @@ -75,6 +114,7 @@ describe('get_local_stats', () => { nodes: { yup: 'abc' }, random: 123, }; + const kibana = { kibana: { great: 'googlymoogly', @@ -97,12 +137,16 @@ describe('get_local_stats', () => { snow: { chances: 0 }, }; + const clusterStatsWithNodesUsage = { + ...clusterStats, + nodes: merge(clusterStats.nodes, { usage: nodesUsage }), + }; const combinedStatsResult = { collection: 'local', cluster_uuid: clusterUuid, cluster_name: clusterName, version, - cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'), + cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'), stack_stats: { kibana: { great: 'googlymoogly', @@ -135,7 +179,7 @@ describe('get_local_stats', () => { describe('handleLocalStats', () => { it('returns expected object without xpack and kibana data', () => { - const result = handleLocalStats(clusterInfo, clusterStats, void 0, context); + const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context); expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); @@ -146,7 +190,7 @@ describe('get_local_stats', () => { }); it('returns expected object with xpack', () => { - const result = handleLocalStats(clusterInfo, clusterStats, void 0, context); + const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context); const { stack_stats: stack, ...cluster } = result; expect(cluster.collection).to.be(combinedStatsResult.collection); expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid); @@ -167,7 +211,8 @@ describe('get_local_stats', () => { mockGetLocalStats( callClusterUsageFailed, Promise.resolve(clusterInfo), - Promise.resolve(clusterStats) + Promise.resolve(clusterStats), + Promise.resolve(nodesUsage) ); const result = await getLocalStats([], { server: getMockServer(), @@ -177,6 +222,7 @@ describe('get_local_stats', () => { expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid); expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name); expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats); + expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes); expect(result.version).to.be('2.3.4'); expect(result.collection).to.be('local'); @@ -188,7 +234,12 @@ describe('get_local_stats', () => { it('returns expected object with xpack and kibana data', async () => { const callCluster = sinon.stub(); const usageCollection = mockUsageCollection(kibana); - mockGetLocalStats(callCluster, Promise.resolve(clusterInfo), Promise.resolve(clusterStats)); + mockGetLocalStats( + callCluster, + Promise.resolve(clusterInfo), + Promise.resolve(clusterStats), + Promise.resolve(nodesUsage) + ); const result = await getLocalStats([], { server: getMockServer(callCluster), diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts index b77d01c5b431fd..b42edde2f55ca2 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.ts @@ -24,6 +24,7 @@ import { import { getClusterInfo, ESClusterInfo } from './get_cluster_info'; import { getClusterStats } from './get_cluster_stats'; import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana'; +import { getNodesUsage } from './get_nodes_usage'; /** * Handle the separate local calls by combining them into a single object response that looks like the @@ -67,12 +68,21 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async ( return await Promise.all( clustersDetails.map(async (clustersDetail) => { - const [clusterInfo, clusterStats, kibana] = await Promise.all([ + const [clusterInfo, clusterStats, nodesUsage, kibana] = await Promise.all([ getClusterInfo(callCluster), // cluster info getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_) + getNodesUsage(callCluster), // nodes_usage info getKibana(usageCollection, callCluster), ]); - return handleLocalStats(clusterInfo, clusterStats, kibana, context); + return handleLocalStats( + clusterInfo, + { + ...clusterStats, + nodes: { ...clusterStats.nodes, usage: nodesUsage }, + }, + kibana, + context + ); }) ); }; diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts new file mode 100644 index 00000000000000..4e4b0e11b79794 --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.test.ts @@ -0,0 +1,80 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { getNodesUsage } from './get_nodes_usage'; +import { TIMEOUT } from './constants'; + +const mockedNodesFetchResponse = { + cluster_name: 'test cluster', + nodes: { + some_node_id: { + timestamp: 1588617023177, + since: 1588616945163, + rest_actions: { + nodes_usage_action: 1, + create_index_action: 1, + document_get_action: 1, + search_action: 19, + nodes_info_action: 36, + }, + aggregations: { + terms: { + bytes: 2, + }, + scripted_metric: { + other: 7, + }, + }, + }, + }, +}; +describe('get_nodes_usage', () => { + it('calls fetchNodesUsage', async () => { + const callCluster = jest.fn(); + callCluster.mockResolvedValueOnce(mockedNodesFetchResponse); + await getNodesUsage(callCluster); + expect(callCluster).toHaveBeenCalledWith('transport.request', { + path: '/_nodes/usage', + method: 'GET', + query: { + timeout: TIMEOUT, + }, + }); + }); + it('returns a modified array of node usage data', async () => { + const callCluster = jest.fn(); + callCluster.mockResolvedValueOnce(mockedNodesFetchResponse); + const result = await getNodesUsage(callCluster); + expect(result.nodes).toEqual([ + { + aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } }, + node_id: 'some_node_id', + rest_actions: { + create_index_action: 1, + document_get_action: 1, + nodes_info_action: 36, + nodes_usage_action: 1, + search_action: 19, + }, + since: 1588616945163, + timestamp: 1588617023177, + }, + ]); + }); +}); diff --git a/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts new file mode 100644 index 00000000000000..c5c110fbb4149b --- /dev/null +++ b/src/plugins/telemetry/server/telemetry_collection/get_nodes_usage.ts @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { LegacyAPICaller } from 'kibana/server'; +import { TIMEOUT } from './constants'; + +export interface NodeAggregation { + [key: string]: number; +} + +// we set aggregations as an optional type because it was only added in v7.8.0 +export interface NodeObj { + node_id?: string; + timestamp: number; + since: number; + rest_actions: { + [key: string]: number; + }; + aggregations?: { + [key: string]: NodeAggregation; + }; +} + +export interface NodesFeatureUsageResponse { + cluster_name: string; + nodes: { + [key: string]: NodeObj; + }; +} + +export type NodesUsageGetter = ( + callCluster: LegacyAPICaller +) => Promise<{ nodes: NodeObj[] | Array<{}> }>; +/** + * Get the nodes usage data from the connected cluster. + * + * This is the equivalent to GET /_nodes/usage?timeout=30s. + * + * The Nodes usage API was introduced in v6.0.0 + */ +export async function fetchNodesUsage( + callCluster: LegacyAPICaller +): Promise { + const response = await callCluster('transport.request', { + method: 'GET', + path: '/_nodes/usage', + query: { + timeout: TIMEOUT, + }, + }); + return response; +} + +/** + * Get the nodes usage from the connected cluster + * @param callCluster APICaller + * @returns Object containing array of modified usage information with the node_id nested within the data for that node. + */ +export const getNodesUsage: NodesUsageGetter = async (callCluster) => { + const result = await fetchNodesUsage(callCluster); + const transformedNodes = Object.entries(result?.nodes || {}).map(([key, value]) => ({ + ...(value as NodeObj), + node_id: key, + })); + return { nodes: transformedNodes }; +}; diff --git a/test/api_integration/apis/telemetry/telemetry_local.js b/test/api_integration/apis/telemetry/telemetry_local.js index d5b8f3fe378c57..b660341d0a9cb6 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.js +++ b/test/api_integration/apis/telemetry/telemetry_local.js @@ -113,6 +113,7 @@ export default function ({ getService }) { 'cluster_stats.nodes.plugins', 'cluster_stats.nodes.process', 'cluster_stats.nodes.versions', + 'cluster_stats.nodes.usage', 'cluster_stats.status', 'cluster_stats.timestamp', 'cluster_uuid', diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap index 1a70504dc93918..ed82dc65eb4108 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/__snapshots__/get_stats_with_xpack.test.ts.snap @@ -4,7 +4,27 @@ exports[`Telemetry Collection: Get Aggregated Stats OSS-like telemetry (no licen Array [ Object { "cluster_name": "test", - "cluster_stats": Object {}, + "cluster_stats": Object { + "nodes": Object { + "usage": Object { + "nodes": Array [ + Object { + "aggregations": Object { + "terms": Object { + "bytes": 2, + }, + }, + "node_id": "some_node_id", + "rest_actions": Object { + "nodes_usage_action": 1, + }, + "since": 1588616945163, + "timestamp": 1588617023177, + }, + ], + }, + }, + }, "cluster_uuid": "test", "collection": "local", "stack_stats": Object { @@ -62,7 +82,27 @@ exports[`Telemetry Collection: Get Aggregated Stats X-Pack telemetry (license + Array [ Object { "cluster_name": "test", - "cluster_stats": Object {}, + "cluster_stats": Object { + "nodes": Object { + "usage": Object { + "nodes": Array [ + Object { + "aggregations": Object { + "terms": Object { + "bytes": 2, + }, + }, + "node_id": "some_node_id", + "rest_actions": Object { + "nodes_usage_action": 1, + }, + "since": 1588616945163, + "timestamp": 1588617023177, + }, + ], + }, + }, + }, "cluster_uuid": "test", "collection": "local", "stack_stats": Object { diff --git a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts index 5dfe3d3e99a7f3..a8311933f05317 100644 --- a/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts +++ b/x-pack/plugins/telemetry_collection_xpack/server/telemetry_collection/get_stats_with_xpack.test.ts @@ -28,6 +28,20 @@ const kibana = { rain: { chances: 2 }, snow: { chances: 0 }, }; +const nodesUsage = { + some_node_id: { + timestamp: 1588617023177, + since: 1588616945163, + rest_actions: { + nodes_usage_action: 1, + }, + aggregations: { + terms: { + bytes: 2, + }, + }, + }, +}; const getContext = () => ({ version: '8675309-snapshot', @@ -47,6 +61,11 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { if (options.path === '/_license' || options.path === '/_xpack/usage') { // eslint-disable-next-line no-throw-literal throw { statusCode: 404 }; + } else if (options.path === '/_nodes/usage') { + return { + cluster_name: 'test cluster', + nodes: nodesUsage, + }; } return {}; case 'info': @@ -81,6 +100,12 @@ describe('Telemetry Collection: Get Aggregated Stats', () => { if (options.path === '/_xpack/usage') { return {}; } + if (options.path === '/_nodes/usage') { + return { + cluster_name: 'test cluster', + nodes: nodesUsage, + }; + } case 'info': return { cluster_uuid: 'test', cluster_name: 'test', version: { number: '8.0.0' } }; default: