Skip to content

Commit

Permalink
[Usage Collection] Report nodes feature usage (#70108) (#70329)
Browse files Browse the repository at this point in the history
* Adds nodes feature usage stats merged into cluster_stats.nodes when usage collection is local
  • Loading branch information
TinaHeiligers authored Jun 30, 2020
1 parent 95da850 commit 6795a4c
Show file tree
Hide file tree
Showing 7 changed files with 299 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@

import expect from '@kbn/expect';
import sinon from 'sinon';
import { merge, omit } from 'lodash';

import { TIMEOUT } from '../constants';
import { mockGetClusterInfo } from './get_cluster_info';
import { mockGetClusterStats } from './get_cluster_stats';

import { omit } from 'lodash';
import { getLocalStats, handleLocalStats } from '../get_local_stats';

const mockUsageCollection = (kibanaUsage = {}) => ({
Expand Down Expand Up @@ -51,10 +52,26 @@ const getMockServer = (getCluster = sinon.stub()) => ({
elasticsearch: { getCluster },
},
});
function mockGetNodesUsage(callCluster, nodesUsage, req) {
callCluster
.withArgs(
req,
{
method: 'GET',
path: '/_nodes/usage',
query: {
timeout: TIMEOUT,
},
},
'transport.request'
)
.returns(nodesUsage);
}

function mockGetLocalStats(callCluster, clusterInfo, clusterStats, req) {
function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) {
mockGetClusterInfo(callCluster, clusterInfo, req);
mockGetClusterStats(callCluster, clusterStats, req);
mockGetNodesUsage(callCluster, nodesUsage, req);
}

describe('get_local_stats', () => {
Expand All @@ -68,13 +85,36 @@ describe('get_local_stats', () => {
number: version,
},
};
const nodesUsage = [
{
node_id: 'some_node_id',
timestamp: 1588617023177,
since: 1588616945163,
rest_actions: {
nodes_usage_action: 1,
create_index_action: 1,
document_get_action: 1,
search_action: 19,
nodes_info_action: 36,
},
aggregations: {
terms: {
bytes: 2,
},
scripted_metric: {
other: 7,
},
},
},
];
const clusterStats = {
_nodes: { failed: 123 },
cluster_name: 'real-cool',
indices: { totally: 456 },
nodes: { yup: 'abc' },
random: 123,
};

const kibana = {
kibana: {
great: 'googlymoogly',
Expand All @@ -97,12 +137,16 @@ describe('get_local_stats', () => {
snow: { chances: 0 },
};

const clusterStatsWithNodesUsage = {
...clusterStats,
nodes: merge(clusterStats.nodes, { usage: nodesUsage }),
};
const combinedStatsResult = {
collection: 'local',
cluster_uuid: clusterUuid,
cluster_name: clusterName,
version,
cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'),
stack_stats: {
kibana: {
great: 'googlymoogly',
Expand Down Expand Up @@ -135,7 +179,7 @@ describe('get_local_stats', () => {

describe('handleLocalStats', () => {
it('returns expected object without xpack and kibana data', () => {
const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
Expand All @@ -146,7 +190,7 @@ describe('get_local_stats', () => {
});

it('returns expected object with xpack', () => {
const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
const { stack_stats: stack, ...cluster } = result;
expect(cluster.collection).to.be(combinedStatsResult.collection);
expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
Expand All @@ -167,7 +211,8 @@ describe('get_local_stats', () => {
mockGetLocalStats(
callClusterUsageFailed,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats)
Promise.resolve(clusterStats),
Promise.resolve(nodesUsage)
);
const result = await getLocalStats([], {
server: getMockServer(),
Expand All @@ -177,6 +222,7 @@ describe('get_local_stats', () => {
expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes);
expect(result.version).to.be('2.3.4');
expect(result.collection).to.be('local');

Expand All @@ -188,7 +234,12 @@ describe('get_local_stats', () => {
it('returns expected object with xpack and kibana data', async () => {
const callCluster = sinon.stub();
const usageCollection = mockUsageCollection(kibana);
mockGetLocalStats(callCluster, Promise.resolve(clusterInfo), Promise.resolve(clusterStats));
mockGetLocalStats(
callCluster,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats),
Promise.resolve(nodesUsage)
);

const result = await getLocalStats([], {
server: getMockServer(callCluster),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import {
import { getClusterInfo, ESClusterInfo } from './get_cluster_info';
import { getClusterStats } from './get_cluster_stats';
import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana';
import { getNodesUsage } from './get_nodes_usage';

/**
* Handle the separate local calls by combining them into a single object response that looks like the
Expand Down Expand Up @@ -67,12 +68,21 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (

return await Promise.all(
clustersDetails.map(async (clustersDetail) => {
const [clusterInfo, clusterStats, kibana] = await Promise.all([
const [clusterInfo, clusterStats, nodesUsage, kibana] = await Promise.all([
getClusterInfo(callCluster), // cluster info
getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
getNodesUsage(callCluster), // nodes_usage info
getKibana(usageCollection, callCluster),
]);
return handleLocalStats(clusterInfo, clusterStats, kibana, context);
return handleLocalStats(
clusterInfo,
{
...clusterStats,
nodes: { ...clusterStats.nodes, usage: nodesUsage },
},
kibana,
context
);
})
);
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import { getNodesUsage } from './get_nodes_usage';
import { TIMEOUT } from './constants';

const mockedNodesFetchResponse = {
cluster_name: 'test cluster',
nodes: {
some_node_id: {
timestamp: 1588617023177,
since: 1588616945163,
rest_actions: {
nodes_usage_action: 1,
create_index_action: 1,
document_get_action: 1,
search_action: 19,
nodes_info_action: 36,
},
aggregations: {
terms: {
bytes: 2,
},
scripted_metric: {
other: 7,
},
},
},
},
};
describe('get_nodes_usage', () => {
it('calls fetchNodesUsage', async () => {
const callCluster = jest.fn();
callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
await getNodesUsage(callCluster);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_nodes/usage',
method: 'GET',
query: {
timeout: TIMEOUT,
},
});
});
it('returns a modified array of node usage data', async () => {
const callCluster = jest.fn();
callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
const result = await getNodesUsage(callCluster);
expect(result.nodes).toEqual([
{
aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } },
node_id: 'some_node_id',
rest_actions: {
create_index_action: 1,
document_get_action: 1,
nodes_info_action: 36,
nodes_usage_action: 1,
search_action: 19,
},
since: 1588616945163,
timestamp: 1588617023177,
},
]);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { LegacyAPICaller } from 'kibana/server';
import { TIMEOUT } from './constants';

export interface NodeAggregation {
[key: string]: number;
}

// we set aggregations as an optional type because it was only added in v7.8.0
export interface NodeObj {
node_id?: string;
timestamp: number;
since: number;
rest_actions: {
[key: string]: number;
};
aggregations?: {
[key: string]: NodeAggregation;
};
}

export interface NodesFeatureUsageResponse {
cluster_name: string;
nodes: {
[key: string]: NodeObj;
};
}

export type NodesUsageGetter = (
callCluster: LegacyAPICaller
) => Promise<{ nodes: NodeObj[] | Array<{}> }>;
/**
* Get the nodes usage data from the connected cluster.
*
* This is the equivalent to GET /_nodes/usage?timeout=30s.
*
* The Nodes usage API was introduced in v6.0.0
*/
export async function fetchNodesUsage(
callCluster: LegacyAPICaller
): Promise<NodesFeatureUsageResponse> {
const response = await callCluster('transport.request', {
method: 'GET',
path: '/_nodes/usage',
query: {
timeout: TIMEOUT,
},
});
return response;
}

/**
* Get the nodes usage from the connected cluster
* @param callCluster APICaller
* @returns Object containing array of modified usage information with the node_id nested within the data for that node.
*/
export const getNodesUsage: NodesUsageGetter = async (callCluster) => {
const result = await fetchNodesUsage(callCluster);
const transformedNodes = Object.entries(result?.nodes || {}).map(([key, value]) => ({
...(value as NodeObj),
node_id: key,
}));
return { nodes: transformedNodes };
};
1 change: 1 addition & 0 deletions test/api_integration/apis/telemetry/telemetry_local.js
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ export default function ({ getService }) {
'cluster_stats.nodes.plugins',
'cluster_stats.nodes.process',
'cluster_stats.nodes.versions',
'cluster_stats.nodes.usage',
'cluster_stats.status',
'cluster_stats.timestamp',
'cluster_uuid',
Expand Down
Loading

0 comments on commit 6795a4c

Please sign in to comment.