Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Usage Collection] Report nodes feature usage #70108

Merged
Merged
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@

import expect from '@kbn/expect';
import sinon from 'sinon';
import { merge, omit } from 'lodash';

import { TIMEOUT } from '../constants';
import { mockGetClusterInfo } from './get_cluster_info';
import { mockGetClusterStats } from './get_cluster_stats';

import { omit } from 'lodash';
import { getLocalStats, handleLocalStats } from '../get_local_stats';

const mockUsageCollection = (kibanaUsage = {}) => ({
Expand Down Expand Up @@ -51,10 +52,26 @@ const getMockServer = (getCluster = sinon.stub()) => ({
elasticsearch: { getCluster },
},
});
function mockGetNodesUsage(callCluster, nodesUsage, req) {
callCluster
.withArgs(
req,
{
method: 'GET',
path: '/_nodes/usage',
query: {
timeout: TIMEOUT,
},
},
'transport.request'
)
.returns(nodesUsage);
}

function mockGetLocalStats(callCluster, clusterInfo, clusterStats, req) {
function mockGetLocalStats(callCluster, clusterInfo, clusterStats, nodesUsage, req) {
mockGetClusterInfo(callCluster, clusterInfo, req);
mockGetClusterStats(callCluster, clusterStats, req);
mockGetNodesUsage(callCluster, nodesUsage, req);
}

describe('get_local_stats', () => {
Expand All @@ -68,13 +85,36 @@ describe('get_local_stats', () => {
number: version,
},
};
const nodesUsage = [
{
node_id: 'some_node_id',
timestamp: 1588617023177,
since: 1588616945163,
rest_actions: {
nodes_usage_action: 1,
create_index_action: 1,
document_get_action: 1,
search_action: 19,
nodes_info_action: 36,
},
aggregations: {
terms: {
bytes: 2,
},
scripted_metric: {
other: 7,
},
},
},
];
const clusterStats = {
_nodes: { failed: 123 },
cluster_name: 'real-cool',
indices: { totally: 456 },
nodes: { yup: 'abc' },
random: 123,
};

const kibana = {
kibana: {
great: 'googlymoogly',
Expand All @@ -97,12 +137,16 @@ describe('get_local_stats', () => {
snow: { chances: 0 },
};

const clusterStatsWithNodesUsage = {
...clusterStats,
nodes: merge(clusterStats.nodes, { usage: nodesUsage }),
};
const combinedStatsResult = {
collection: 'local',
cluster_uuid: clusterUuid,
cluster_name: clusterName,
version,
cluster_stats: omit(clusterStats, '_nodes', 'cluster_name'),
cluster_stats: omit(clusterStatsWithNodesUsage, '_nodes', 'cluster_name'),
stack_stats: {
kibana: {
great: 'googlymoogly',
Expand Down Expand Up @@ -135,7 +179,7 @@ describe('get_local_stats', () => {

describe('handleLocalStats', () => {
it('returns expected object without xpack and kibana data', () => {
const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
Expand All @@ -146,7 +190,7 @@ describe('get_local_stats', () => {
});

it('returns expected object with xpack', () => {
const result = handleLocalStats(clusterInfo, clusterStats, void 0, context);
const result = handleLocalStats(clusterInfo, clusterStatsWithNodesUsage, void 0, context);
const { stack_stats: stack, ...cluster } = result;
expect(cluster.collection).to.be(combinedStatsResult.collection);
expect(cluster.cluster_uuid).to.be(combinedStatsResult.cluster_uuid);
Expand All @@ -167,7 +211,8 @@ describe('get_local_stats', () => {
mockGetLocalStats(
callClusterUsageFailed,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats)
Promise.resolve(clusterStats),
Promise.resolve(nodesUsage)
);
const result = await getLocalStats([], {
server: getMockServer(),
Expand All @@ -177,6 +222,7 @@ describe('get_local_stats', () => {
expect(result.cluster_uuid).to.eql(combinedStatsResult.cluster_uuid);
expect(result.cluster_name).to.eql(combinedStatsResult.cluster_name);
expect(result.cluster_stats).to.eql(combinedStatsResult.cluster_stats);
expect(result.cluster_stats.nodes).to.eql(combinedStatsResult.cluster_stats.nodes);
expect(result.version).to.be('2.3.4');
expect(result.collection).to.be('local');

Expand All @@ -188,7 +234,12 @@ describe('get_local_stats', () => {
it('returns expected object with xpack and kibana data', async () => {
const callCluster = sinon.stub();
const usageCollection = mockUsageCollection(kibana);
mockGetLocalStats(callCluster, Promise.resolve(clusterInfo), Promise.resolve(clusterStats));
mockGetLocalStats(
callCluster,
Promise.resolve(clusterInfo),
Promise.resolve(clusterStats),
Promise.resolve(nodesUsage)
);

const result = await getLocalStats([], {
server: getMockServer(callCluster),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ import {
StatsGetter,
StatsCollectionContext,
} from 'src/plugins/telemetry_collection_manager/server';
import { merge } from 'lodash';
import { getClusterInfo, ESClusterInfo } from './get_cluster_info';
import { getClusterStats } from './get_cluster_stats';
import { getKibana, handleKibanaStats, KibanaUsageStats } from './get_kibana';
import { getNodesUsage } from './get_nodes_usage';

/**
* Handle the separate local calls by combining them into a single object response that looks like the
Expand Down Expand Up @@ -67,12 +69,21 @@ export const getLocalStats: StatsGetter<{}, TelemetryLocalStats> = async (

return await Promise.all(
clustersDetails.map(async (clustersDetail) => {
const [clusterInfo, clusterStats, kibana] = await Promise.all([
const [clusterInfo, clusterStats, nodesUsage, kibana] = await Promise.all([
getClusterInfo(callCluster), // cluster info
getClusterStats(callCluster), // cluster stats (not to be confused with cluster _state_)
getNodesUsage(callCluster), // nodes_usage info
getKibana(usageCollection, callCluster),
]);
return handleLocalStats(clusterInfo, clusterStats, kibana, context);
return handleLocalStats(
clusterInfo,
{
...clusterStats,
nodes: merge(clusterStats.nodes, { usage: nodesUsage }),
TinaHeiligers marked this conversation as resolved.
Show resolved Hide resolved
},
kibana,
context
);
})
);
};
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import { getNodesUsage } from './get_nodes_usage';
import { TIMEOUT } from './constants';

const mockedNodesResponse = {
node_id: 'some_node_id',
timestamp: 1588617023177,
since: 1588616945163,
rest_actions: {
nodes_usage_action: 1,
create_index_action: 1,
document_get_action: 1,
search_action: 19,
nodes_info_action: 36,
},
aggregations: {
terms: {
bytes: 2,
},
scripted_metric: {
other: 7,
},
},
};
const mockedNodesFetchResponse = {
cluster_name: 'test cluster',
nodes: {
some_node_id: {
timestamp: 1588617023177,
since: 1588616945163,
rest_actions: {
nodes_usage_action: 1,
create_index_action: 1,
document_get_action: 1,
search_action: 19,
nodes_info_action: 36,
},
aggregations: {
terms: {
bytes: 2,
},
scripted_metric: {
other: 7,
},
},
},
},
};
describe('get_nodes_usage', () => {
it('calls fetchNodesUsage', async () => {
const callCluster = jest.fn();
callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
jest.fn(async () => ({
cluster_name: 'test',
nodes: mockedNodesResponse,
}));
TinaHeiligers marked this conversation as resolved.
Show resolved Hide resolved
await getNodesUsage(callCluster);
expect(callCluster).toHaveBeenCalledWith('transport.request', {
path: '/_nodes/usage',
method: 'GET',
query: {
timeout: TIMEOUT,
},
});
});
it('returns a modified array of node usage data', async () => {
const callCluster = jest.fn();
callCluster.mockResolvedValueOnce(mockedNodesFetchResponse);
jest.fn(async () => ({
cluster_name: 'test',
nodes: mockedNodesResponse,
}));
const result = await getNodesUsage(callCluster);
expect(result.nodes).toEqual([
{
aggregations: { scripted_metric: { other: 7 }, terms: { bytes: 2 } },
node_id: 'some_node_id',
rest_actions: {
create_index_action: 1,
document_get_action: 1,
nodes_info_action: 36,
nodes_usage_action: 1,
search_action: 19,
},
since: 1588616945163,
timestamp: 1588617023177,
},
]);
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Licensed to Elasticsearch B.V. under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch B.V. licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import { APICaller } from 'kibana/server';
import { TIMEOUT } from './constants';

export interface NodeAggregation {
[key: string]: number;
}

// we set aggregations as an optional type because it was only added in v7.8.0
export interface NodeObj {
node_id?: string;
timestamp: number;
since: number;
rest_actions: {
[key: string]: number;
};
aggregations?: {
[key: string]: NodeAggregation;
};
}
export interface NodesStats {
total: number;
successful: number;
failed: number;
}
TinaHeiligers marked this conversation as resolved.
Show resolved Hide resolved
export interface NodesFeatureUsageResponse {
cluster_name: string;
nodes: {
[key: string]: NodeObj;
};
}

export type NodesUsageFetcher = (callCluster: APICaller) => Promise<NodesFeatureUsageResponse>;

export type NodesUsageGetter = (
callCluster: APICaller
) => Promise<{ nodes: NodeObj[] | Array<{}> }>;
/**
* Get the nodes usage data from the connected cluster.
*
* This is the equivalent to GET /_nodes/usage?timeout=30s.
*
* The Nodes usage API was introduced in v6.0.0
*/
export async function fetchNodesUsage<NodesUsageFetcher>(callCluster: APICaller) {
TinaHeiligers marked this conversation as resolved.
Show resolved Hide resolved
const response = await callCluster('transport.request', {
method: 'GET',
path: '/_nodes/usage',
query: {
timeout: TIMEOUT,
},
});
return response;
}

/**
* Get the nodes usage from the connected cluster
* @param callCluster APICaller
* @returns Object containing array of modified usage information with the node_id nested within the data for that node.
*/
export const getNodesUsage: NodesUsageGetter = async (callCluster) => {
const result = await fetchNodesUsage<NodesUsageFetcher>(callCluster);
let transformedNodes = [{}];
TinaHeiligers marked this conversation as resolved.
Show resolved Hide resolved
if (result && result.nodes) {
transformedNodes = Object.entries(result.nodes).map(([key, value]) => ({
...(value as NodeObj),
node_id: key,
}));
}
return { nodes: transformedNodes };
};
1 change: 1 addition & 0 deletions test/api_integration/apis/telemetry/telemetry_local.js
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ export default function ({ getService }) {
'cluster_stats.nodes.plugins',
'cluster_stats.nodes.process',
'cluster_stats.nodes.versions',
'cluster_stats.nodes.usage',
'cluster_stats.status',
'cluster_stats.timestamp',
'cluster_uuid',
Expand Down
Loading