From 1af551bae1adb430a4cc4d6aeaf7458817210a46 Mon Sep 17 00:00:00 2001 From: Ahmad Bamieh Date: Mon, 2 Nov 2020 13:51:18 +0200 Subject: [PATCH] [Telemetry] Remove `from` and `to` timestamps from usage stats APIs (#81579) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../public/services/telemetry_service.test.ts | 30 +- .../public/services/telemetry_service.ts | 9 +- src/plugins/telemetry/server/fetcher.ts | 3 +- .../server/routes/telemetry_opt_in.ts | 3 +- .../server/routes/telemetry_opt_in_stats.ts | 3 +- .../server/routes/telemetry_usage_stats.ts | 17 +- .../get_local_stats.test.ts | 3 +- .../server/plugin.ts | 4 +- .../server/types.ts | 6 +- .../apis/telemetry/telemetry_local.js | 23 +- .../api_debug/apis/telemetry/index.js | 14 +- x-pack/plugins/monitoring/common/constants.ts | 16 + .../get_all_stats.test.ts | 9 +- .../telemetry_collection/get_all_stats.ts | 25 +- .../get_beats_stats.test.ts | 4 +- .../telemetry_collection/get_beats_stats.ts | 22 +- .../get_cluster_uuids.test.ts | 7 +- .../telemetry_collection/get_cluster_uuids.ts | 17 +- .../get_high_level_stats.test.ts | 4 +- .../get_high_level_stats.ts | 10 +- .../telemetry_collection/get_kibana_stats.ts | 22 +- .../apis/telemetry/fixtures/multicluster.json | 1253 +++++++++-------- .../apis/telemetry/telemetry.js | 17 +- .../apis/telemetry/telemetry_local.js | 14 +- .../api_integration/services/usage_api.ts | 2 +- .../test/functional/apps/infra/home_page.ts | 10 +- .../apps/infra/logs_source_configuration.ts | 5 +- 27 files changed, 770 insertions(+), 782 deletions(-) diff --git a/src/plugins/telemetry/public/services/telemetry_service.test.ts b/src/plugins/telemetry/public/services/telemetry_service.test.ts index 8bb6fe6d50be08..ba8b5e1e46f58d 100644 --- a/src/plugins/telemetry/public/services/telemetry_service.test.ts +++ b/src/plugins/telemetry/public/services/telemetry_service.test.ts @@ -18,42 +18,30 @@ */ /* eslint-disable dot-notation */ -import { mockTelemetryService } from '../mocks'; - -const mockSubtract = jest.fn().mockImplementation(() => { - return { - toISOString: jest.fn(), - }; -}); - -const mockClone = jest.fn().mockImplementation(() => { - return { - clone: mockClone, - subtract: mockSubtract, - toISOString: jest.fn(), - }; -}); +const mockMomentValueOf = jest.fn(); jest.mock('moment', () => { return jest.fn().mockImplementation(() => { return { - clone: mockClone, - subtract: mockSubtract, - toISOString: jest.fn(), + valueOf: mockMomentValueOf, }; }); }); +import { mockTelemetryService } from '../mocks'; + describe('TelemetryService', () => { describe('fetchTelemetry', () => { it('calls expected URL with 20 minutes - now', async () => { + const timestamp = Date.now(); + mockMomentValueOf.mockReturnValueOnce(timestamp); const telemetryService = mockTelemetryService(); + await telemetryService.fetchTelemetry(); expect(telemetryService['http'].post).toBeCalledWith('/api/telemetry/v2/clusters/_stats', { - body: JSON.stringify({ unencrypted: false, timeRange: {} }), + body: JSON.stringify({ unencrypted: false, timestamp }), }); - expect(mockClone).toBeCalled(); - expect(mockSubtract).toBeCalledWith(20, 'minutes'); + expect(mockMomentValueOf).toBeCalled(); }); }); diff --git a/src/plugins/telemetry/public/services/telemetry_service.ts b/src/plugins/telemetry/public/services/telemetry_service.ts index 658f02bfba3954..fade614be6c65e 100644 --- a/src/plugins/telemetry/public/services/telemetry_service.ts +++ b/src/plugins/telemetry/public/services/telemetry_service.ts @@ -121,17 +121,10 @@ export class TelemetryService { }; public fetchTelemetry = async ({ unencrypted = false } = {}) => { - const now = moment(); return this.http.post('/api/telemetry/v2/clusters/_stats', { body: JSON.stringify({ unencrypted, - timeRange: { - min: now - .clone() // Need to clone it to avoid mutation (and max being the same value) - .subtract(20, 'minutes') - .toISOString(), - max: now.toISOString(), - }, + timestamp: moment().valueOf(), }), }); }; diff --git a/src/plugins/telemetry/server/fetcher.ts b/src/plugins/telemetry/server/fetcher.ts index 8d865cd20cde12..a3649f51577ac3 100644 --- a/src/plugins/telemetry/server/fetcher.ts +++ b/src/plugins/telemetry/server/fetcher.ts @@ -213,8 +213,7 @@ export class FetcherTask { private async fetchTelemetry() { return await this.telemetryCollectionManager!.getStats({ unencrypted: false, - start: moment().subtract(20, 'minutes').toISOString(), - end: moment().toISOString(), + timestamp: moment().valueOf(), }); } diff --git a/src/plugins/telemetry/server/routes/telemetry_opt_in.ts b/src/plugins/telemetry/server/routes/telemetry_opt_in.ts index b9a6d104892000..83a35b22d2f307 100644 --- a/src/plugins/telemetry/server/routes/telemetry_opt_in.ts +++ b/src/plugins/telemetry/server/routes/telemetry_opt_in.ts @@ -86,8 +86,7 @@ export function registerTelemetryOptInRoutes({ } const statsGetterConfig: StatsGetterConfig = { - start: moment().subtract(20, 'minutes').toISOString(), - end: moment().toISOString(), + timestamp: moment().valueOf(), unencrypted: false, }; diff --git a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts index 6a217085b13e7a..ff2e0be1aa1aeb 100644 --- a/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts +++ b/src/plugins/telemetry/server/routes/telemetry_opt_in_stats.ts @@ -72,8 +72,7 @@ export function registerTelemetryOptInStatsRoutes( const unencrypted = req.body.unencrypted; const statsGetterConfig: StatsGetterConfig = { - start: moment().subtract(20, 'minutes').toISOString(), - end: moment().toISOString(), + timestamp: moment().valueOf(), unencrypted, request: req, }; diff --git a/src/plugins/telemetry/server/routes/telemetry_usage_stats.ts b/src/plugins/telemetry/server/routes/telemetry_usage_stats.ts index d47a2a0780efea..fd888cbe087f75 100644 --- a/src/plugins/telemetry/server/routes/telemetry_usage_stats.ts +++ b/src/plugins/telemetry/server/routes/telemetry_usage_stats.ts @@ -32,8 +32,6 @@ const validate: TypeOptions['validate'] = (value) => { } }; -const dateSchema = schema.oneOf([schema.string({ validate }), schema.number({ validate })]); - export function registerTelemetryUsageStatsRoutes( router: IRouter, telemetryCollectionManager: TelemetryCollectionManagerPluginSetup, @@ -45,25 +43,20 @@ export function registerTelemetryUsageStatsRoutes( validate: { body: schema.object({ unencrypted: schema.boolean({ defaultValue: false }), - timeRange: schema.object({ - min: dateSchema, - max: dateSchema, - }), + timestamp: schema.oneOf([schema.string({ validate }), schema.number({ validate })]), }), }, }, async (context, req, res) => { - const start = moment(req.body.timeRange.min).toISOString(); - const end = moment(req.body.timeRange.max).toISOString(); - const unencrypted = req.body.unencrypted; + const { unencrypted, timestamp } = req.body; try { const statsConfig: StatsGetterConfig = { - unencrypted, - start, - end, + timestamp: moment(timestamp).valueOf(), request: req, + unencrypted, }; + const stats = await telemetryCollectionManager.getStats(statsConfig); return res.ok({ body: stats }); } catch (err) { diff --git a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts index fcecbca23038ef..9298b36ac3ea66 100644 --- a/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts +++ b/src/plugins/telemetry/server/telemetry_collection/get_local_stats.test.ts @@ -87,8 +87,7 @@ function mockStatsCollectionConfig(clusterInfo: any, clusterStats: any, kibana: ...createCollectorFetchContextMock(), esClient: mockGetLocalStats(clusterInfo, clusterStats), usageCollection: mockUsageCollection(kibana), - start: '', - end: '', + timestamp: Date.now(), }; } diff --git a/src/plugins/telemetry_collection_manager/server/plugin.ts b/src/plugins/telemetry_collection_manager/server/plugin.ts index 4900e75a1936ba..e1e1379097adf8 100644 --- a/src/plugins/telemetry_collection_manager/server/plugin.ts +++ b/src/plugins/telemetry_collection_manager/server/plugin.ts @@ -144,7 +144,7 @@ export class TelemetryCollectionManagerPlugin collectionSoService: SavedObjectsServiceStart, usageCollection: UsageCollectionSetup ): StatsCollectionConfig { - const { start, end, request } = config; + const { timestamp, request } = config; const callCluster = config.unencrypted ? collection.esCluster.asScoped(request).callAsCurrentUser @@ -157,7 +157,7 @@ export class TelemetryCollectionManagerPlugin const soClient = config.unencrypted ? collectionSoService.getScopedClient(config.request) : collectionSoService.createInternalRepository(); - return { callCluster, start, end, usageCollection, esClient, soClient }; + return { callCluster, timestamp, usageCollection, esClient, soClient }; } private async getOptInStats(optInStatus: boolean, config: StatsGetterConfig) { diff --git a/src/plugins/telemetry_collection_manager/server/types.ts b/src/plugins/telemetry_collection_manager/server/types.ts index d6e4fdce2b188a..0100fdbbb39700 100644 --- a/src/plugins/telemetry_collection_manager/server/types.ts +++ b/src/plugins/telemetry_collection_manager/server/types.ts @@ -56,8 +56,7 @@ export interface TelemetryOptInStats { export interface BaseStatsGetterConfig { unencrypted: boolean; - start: string; - end: string; + timestamp: number; request?: KibanaRequest; } @@ -77,8 +76,7 @@ export interface ClusterDetails { export interface StatsCollectionConfig { usageCollection: UsageCollectionSetup; callCluster: LegacyAPICaller; - start: string | number; - end: string | number; + timestamp: number; esClient: ElasticsearchClient; soClient: SavedObjectsClientContract | ISavedObjectsRepository; } diff --git a/test/api_integration/apis/telemetry/telemetry_local.js b/test/api_integration/apis/telemetry/telemetry_local.js index 9a5467e622ff3e..854b462be7704e 100644 --- a/test/api_integration/apis/telemetry/telemetry_local.js +++ b/test/api_integration/apis/telemetry/telemetry_local.js @@ -53,15 +53,12 @@ export default function ({ getService }) { }); it('should pull local stats and validate data types', async () => { - const timeRange = { - min: '2018-07-23T22:07:00Z', - max: '2018-07-23T22:13:00Z', - }; + const timestamp = '2018-07-23T22:13:00Z'; const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); expect(body.length).to.be(1); @@ -98,15 +95,12 @@ export default function ({ getService }) { }); it('should pull local stats and validate fields', async () => { - const timeRange = { - min: '2018-07-23T22:07:00Z', - max: '2018-07-23T22:13:00Z', - }; + const timestamp = '2018-07-23T22:13:00Z'; const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); const stats = body[0]; @@ -156,10 +150,7 @@ export default function ({ getService }) { }); describe('application usage limits', () => { - const timeRange = { - min: '2018-07-23T22:07:00Z', - max: '2018-07-23T22:13:00Z', - }; + const timestamp = '2018-07-23T22:13:00Z'; function createSavedObject() { return supertest @@ -191,7 +182,7 @@ export default function ({ getService }) { const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); expect(body.length).to.be(1); @@ -242,7 +233,7 @@ export default function ({ getService }) { const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); expect(body.length).to.be(1); diff --git a/x-pack/dev-tools/api_debug/apis/telemetry/index.js b/x-pack/dev-tools/api_debug/apis/telemetry/index.js index 79b8c643b61fad..5cee4cdf45d6d5 100644 --- a/x-pack/dev-tools/api_debug/apis/telemetry/index.js +++ b/x-pack/dev-tools/api_debug/apis/telemetry/index.js @@ -7,18 +7,8 @@ import moment from 'moment'; export const name = 'telemetry'; -export const description = 'Get the clusters stats for the last 1 hour from the Kibana server'; +export const description = 'Get the clusters stats from the Kibana server'; export const method = 'POST'; export const path = '/api/telemetry/v2/clusters/_stats'; -// Get an object with start and end times for the last 1 hour, ISO format, in UTC -function getTimeRange() { - const end = moment(); - const start = moment(end).subtract(1, 'hour'); - return { - min: moment.utc(start).format(), - max: moment.utc(end).format(), - }; -} - -export const body = { timeRange: getTimeRange(), unencrypted: true }; +export const body = { timeRange: moment().valueOf(), unencrypted: true }; diff --git a/x-pack/plugins/monitoring/common/constants.ts b/x-pack/plugins/monitoring/common/constants.ts index 8ff8381d702e83..355390100ac2bb 100644 --- a/x-pack/plugins/monitoring/common/constants.ts +++ b/x-pack/plugins/monitoring/common/constants.ts @@ -219,6 +219,22 @@ export const REPORTING_SYSTEM_ID = 'reporting'; */ export const TELEMETRY_COLLECTION_INTERVAL = 86400000; +/** + * The amount of time, in milliseconds, to fetch the cluster uuids from es. + * + * Currently 3 hours. + * @type {Number} + */ +export const CLUSTER_DETAILS_FETCH_INTERVAL = 10800000; + +/** + * The amount of time, in milliseconds, to fetch the usage data from es. + * + * Currently 20 minutes. + * @type {Number} + */ +export const USAGE_FETCH_INTERVAL = 1200000; + /** * The prefix for all alert types used by monitoring */ diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts index 129b7987408069..099f6915611cb7 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.test.ts @@ -12,8 +12,7 @@ import { ClustersHighLevelStats } from './get_high_level_stats'; import { coreMock } from 'src/core/server/mocks'; describe('get_all_stats', () => { - const start = 0; - const end = 1; + const timestamp = Date.now(); const callCluster = sinon.stub(); const esClient = sinon.stub(); const soClient = sinon.stub(); @@ -181,8 +180,7 @@ describe('get_all_stats', () => { esClient: esClient as any, soClient: soClient as any, usageCollection: {} as any, - start, - end, + timestamp, }, { logger: coreMock.createPluginInitializerContext().logger.get('test'), @@ -208,8 +206,7 @@ describe('get_all_stats', () => { esClient: esClient as any, soClient: soClient as any, usageCollection: {} as any, - start, - end, + timestamp, }, { logger: coreMock.createPluginInitializerContext().logger.get('test'), diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts index 9ebd73ffbc8336..b6b2023b2af1ac 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_all_stats.ts @@ -8,15 +8,17 @@ import { set } from '@elastic/safer-lodash-set'; import { get, merge } from 'lodash'; import { StatsGetter } from 'src/plugins/telemetry_collection_manager/server'; -import { LOGSTASH_SYSTEM_ID, KIBANA_SYSTEM_ID, BEATS_SYSTEM_ID } from '../../common/constants'; +import moment from 'moment'; +import { + LOGSTASH_SYSTEM_ID, + KIBANA_SYSTEM_ID, + BEATS_SYSTEM_ID, + USAGE_FETCH_INTERVAL, +} from '../../common/constants'; import { getElasticsearchStats, ESClusterStats } from './get_es_stats'; import { getKibanaStats, KibanaStats } from './get_kibana_stats'; -import { getBeatsStats } from './get_beats_stats'; -import { getHighLevelStats } from './get_high_level_stats'; - -type PromiseReturnType any> = ReturnType extends Promise - ? R - : T; +import { getBeatsStats, BeatsStatsByClusterUuid } from './get_beats_stats'; +import { getHighLevelStats, ClustersHighLevelStats } from './get_high_level_stats'; export interface CustomContext { maxBucketSize: number; @@ -28,9 +30,12 @@ export interface CustomContext { */ export const getAllStats: StatsGetter = async ( clustersDetails, - { callCluster, start, end, esClient, soClient }, + { callCluster, timestamp }, { maxBucketSize } ) => { + const start = moment(timestamp).subtract(USAGE_FETCH_INTERVAL, 'ms').toISOString(); + const end = moment(timestamp).toISOString(); + const clusterUuids = clustersDetails.map((clusterDetails) => clusterDetails.clusterUuid); const [esClusters, kibana, logstash, beats] = await Promise.all([ @@ -61,8 +66,8 @@ export function handleAllStats( beats, }: { kibana: KibanaStats; - logstash: PromiseReturnType; - beats: PromiseReturnType; + logstash: ClustersHighLevelStats; + beats: BeatsStatsByClusterUuid; } ) { return clusters.map((cluster) => { diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.test.ts index c619db90c89754..30b3bcdf63adfb 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.test.ts @@ -20,8 +20,8 @@ const getBaseOptions = () => ({ describe('Get Beats Stats', () => { describe('fetchBeatsStats', () => { const clusterUuids = ['aCluster', 'bCluster', 'cCluster']; - const start = 100; - const end = 200; + const start = new Date().toISOString(); + const end = new Date().toISOString(); let callCluster = sinon.stub(); beforeEach(() => { diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts index d153c40bbe58bd..a7f001f166d7d2 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_beats_stats.ts @@ -320,8 +320,8 @@ export function processResults( async function fetchBeatsByType( callCluster: StatsCollectionConfig['callCluster'], clusterUuids: string[], - start: StatsCollectionConfig['start'], - end: StatsCollectionConfig['end'], + start: string, + end: string, { page = 0, ...options }: { page?: number } & BeatsProcessOptions, type: string ): Promise { @@ -384,8 +384,8 @@ async function fetchBeatsByType( export async function fetchBeatsStats( callCluster: StatsCollectionConfig['callCluster'], clusterUuids: string[], - start: StatsCollectionConfig['start'], - end: StatsCollectionConfig['end'], + start: string, + end: string, options: { page?: number } & BeatsProcessOptions ) { return fetchBeatsByType(callCluster, clusterUuids, start, end, options, 'beats_stats'); @@ -394,13 +394,17 @@ export async function fetchBeatsStats( export async function fetchBeatsStates( callCluster: StatsCollectionConfig['callCluster'], clusterUuids: string[], - start: StatsCollectionConfig['start'], - end: StatsCollectionConfig['end'], + start: string, + end: string, options: { page?: number } & BeatsProcessOptions ) { return fetchBeatsByType(callCluster, clusterUuids, start, end, options, 'beats_state'); } +export interface BeatsStatsByClusterUuid { + [clusterUuid: string]: BeatsBaseStats; +} + /* * Call the function for fetching and summarizing beats stats * @return {Object} - Beats stats in an object keyed by the cluster UUIDs @@ -408,9 +412,9 @@ export async function fetchBeatsStates( export async function getBeatsStats( callCluster: StatsCollectionConfig['callCluster'], clusterUuids: string[], - start: StatsCollectionConfig['start'], - end: StatsCollectionConfig['end'] -) { + start: string, + end: string +): Promise { const options: BeatsProcessOptions = { clusters: {}, // the result object to be built up clusterHostSets: {}, // passed to processResults for tracking state in the results generation diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts index c885bc9be44089..0acdb9968bc034 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.test.ts @@ -26,15 +26,14 @@ describe('get_cluster_uuids', () => { const expectedUuids = response.aggregations.cluster_uuids.buckets .map((bucket) => bucket.key) .map((expectedUuid) => ({ clusterUuid: expectedUuid })); - const start = new Date().toISOString(); - const end = new Date().toISOString(); + const timestamp = Date.now(); describe('getClusterUuids', () => { it('returns cluster UUIDs', async () => { callCluster.withArgs('search').returns(Promise.resolve(response)); expect( await getClusterUuids( - { callCluster, esClient, soClient, start, end, usageCollection: {} as any }, + { callCluster, esClient, soClient, timestamp, usageCollection: {} as any }, { maxBucketSize: 1, } as any @@ -48,7 +47,7 @@ describe('get_cluster_uuids', () => { callCluster.returns(Promise.resolve(response)); expect( await fetchClusterUuids( - { callCluster, esClient, soClient, start, end, usageCollection: {} as any }, + { callCluster, esClient, soClient, timestamp, usageCollection: {} as any }, { maxBucketSize: 1, } as any diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts index 9d4657b0ad7995..5f471851b6621c 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_cluster_uuids.ts @@ -5,15 +5,18 @@ */ import { get } from 'lodash'; +import moment from 'moment'; import { ClusterDetailsGetter, StatsCollectionConfig, ClusterDetails, } from 'src/plugins/telemetry_collection_manager/server'; import { createQuery } from './create_query'; -import { INDEX_PATTERN_ELASTICSEARCH } from '../../common/constants'; +import { + INDEX_PATTERN_ELASTICSEARCH, + CLUSTER_DETAILS_FETCH_INTERVAL, +} from '../../common/constants'; import { CustomContext } from './get_all_stats'; - /** * Get a list of Cluster UUIDs that exist within the specified timespan. */ @@ -28,10 +31,14 @@ export const getClusterUuids: ClusterDetailsGetter = async ( /** * Fetch the aggregated Cluster UUIDs from the monitoring cluster. */ -export function fetchClusterUuids( - { callCluster, start, end }: StatsCollectionConfig, +export async function fetchClusterUuids( + { callCluster, timestamp }: StatsCollectionConfig, maxBucketSize: number ) { + const start = moment(timestamp).subtract(CLUSTER_DETAILS_FETCH_INTERVAL, 'ms').toISOString(); + + const end = moment(timestamp).toISOString(); + const params = { index: INDEX_PATTERN_ELASTICSEARCH, size: 0, @@ -50,7 +57,7 @@ export function fetchClusterUuids( }, }; - return callCluster('search', params); + return await callCluster('search', params); } /** diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.test.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.test.ts index d308ddc13e40c3..d7a152d7b54cdb 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.test.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.test.ts @@ -15,8 +15,8 @@ describe('get_high_level_stats', () => { const callWith = sinon.stub(); const product = 'xyz'; const cloudName = 'bare-metal'; - const start = 0; - const end = 1; + const start = new Date().toISOString(); + const end = new Date().toISOString(); const response = { hits: { hits: [ diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts index 481afc86fd115c..ce1c6ccd4b1065 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_high_level_stats.ts @@ -249,8 +249,8 @@ function getIndexPatternForStackProduct(product: string) { export async function getHighLevelStats( callCluster: StatsCollectionConfig['callCluster'], clusterUuids: string[], - start: StatsCollectionConfig['start'], - end: StatsCollectionConfig['end'], + start: string, + end: string, product: string, maxBucketSize: number ) { @@ -270,8 +270,8 @@ export async function fetchHighLevelStats< >( callCluster: StatsCollectionConfig['callCluster'], clusterUuids: string[], - start: StatsCollectionConfig['start'] | undefined, - end: StatsCollectionConfig['end'] | undefined, + start: string, + end: string, product: string, maxBucketSize: number ): Promise> { @@ -347,7 +347,7 @@ export async function fetchHighLevelStats< export function handleHighLevelStatsResponse( response: SearchResponse<{ cluster_uuid?: string }>, product: string -) { +): ClustersHighLevelStats { const instances = response.hits?.hits || []; const clusterMap = groupInstancesByCluster(instances, product); diff --git a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts index e87c8398ad0b01..3bf1d087b973a7 100644 --- a/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts +++ b/x-pack/plugins/monitoring/server/telemetry_collection/get_kibana_stats.ts @@ -147,13 +147,21 @@ export function combineStats( * Ensure the start and end dates are, at least, TELEMETRY_COLLECTION_INTERVAL apart * because, otherwise, we are sending telemetry with empty Kibana usage data. * - * @param {date} [start] The start time from which to get the telemetry data - * @param {date} [end] The end time from which to get the telemetry data + * @param {string} [start] The start time (in ISO string format) from which to get the telemetry data + * @param {string} [end] The end time (in ISO string format) from which to get the telemetry data */ +export function ensureTimeSpan(start: string, end: string): { start: string; end: string }; +export function ensureTimeSpan(start: string, end: undefined): { start: string; end: undefined }; +export function ensureTimeSpan(start: undefined, end: string): { start: undefined; end: string }; export function ensureTimeSpan( - start?: StatsCollectionConfig['start'], - end?: StatsCollectionConfig['end'] -) { + start: undefined, + end: undefined +): { start: undefined; end: undefined }; + +export function ensureTimeSpan( + start?: string, + end?: string +): { start: string | undefined; end: string | undefined } { // We only care if we have a start date, because that's the limit that might make us lose the document if (start) { const duration = moment.duration(TELEMETRY_COLLECTION_INTERVAL, 'milliseconds'); @@ -177,8 +185,8 @@ export function ensureTimeSpan( export async function getKibanaStats( callCluster: StatsCollectionConfig['callCluster'], clusterUuids: string[], - start: StatsCollectionConfig['start'], - end: StatsCollectionConfig['end'], + start: string, + end: string, maxBucketSize: number ) { const { start: safeStart, end: safeEnd } = ensureTimeSpan(start, end); diff --git a/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json b/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json index 7d408e39247ee4..90335d303a36d8 100644 --- a/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json +++ b/x-pack/test/api_integration/apis/telemetry/fixtures/multicluster.json @@ -1,185 +1,244 @@ [ { - "cluster_uuid": "6d-9tDFTRe-qT5GoBytdlQ", - "collectionSource": "monitoring", - "timestamp": "2017-08-15T22:10:59.952Z", - "cluster_name": "clustertwo", - "version": "7.0.0-alpha1", "license": { - "status": "active", - "type": "basic", - "issue_date": "2014-09-29T00:00:00.000Z", "expiry_date": "2030-08-29T23:59:59.999Z", "expiry_date_in_millis": 1914278399999, + "hkey": null, + "issue_date": "2014-09-29T00:00:00.000Z", "issue_date_in_millis": 1411948800000, "issued_to": "issuedTo", "issuer": "issuer", "max_nodes": 1, - "uid": "893361dc-9749-4997-93cb-802e3d7fa4a8", - "hkey": null + "status": "active", + "type": "basic", + "uid": "893361dc-9749-4997-93cb-802e3d7fa4a8" }, + "cluster_name": "clustertwo", "cluster_stats": { - "timestamp": 1502835059952, - "status": "green", "indices": { - "count": 1, - "shards": { - "total": 1, - "primaries": 1, - "replication": 0, - "index": { - "shards": { - "min": 1, - "max": 1, - "avg": 1 - }, - "primaries": { - "min": 1, - "max": 1, - "avg": 1 - }, - "replication": { - "min": 0, - "max": 0, - "avg": 0 - } - } + "completion": { + "size_in_bytes": 0 }, + "count": 1, "docs": { "count": 8, "deleted": 0 }, - "store": { - "size_in_bytes": 34095 - }, "fielddata": { - "memory_size_in_bytes": 0, - "evictions": 0 + "evictions": 0, + "memory_size_in_bytes": 0 }, "query_cache": { - "memory_size_in_bytes": 0, - "total_count": 0, + "cache_count": 0, + "cache_size": 0, + "evictions": 0, "hit_count": 0, + "memory_size_in_bytes": 0, "miss_count": 0, - "cache_size": 0, - "cache_count": 0, - "evictions": 0 - }, - "completion": { - "size_in_bytes": 0 + "total_count": 0 }, "segments": { "count": 8, - "memory_in_bytes": 13852, - "terms_memory_in_bytes": 10412, - "stored_fields_memory_in_bytes": 2496, - "term_vectors_memory_in_bytes": 0, - "norms_memory_in_bytes": 256, - "points_memory_in_bytes": 8, "doc_values_memory_in_bytes": 680, - "index_writer_memory_in_bytes": 0, - "version_map_memory_in_bytes": 0, + "file_sizes": {}, "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 0, "max_unsafe_auto_id_timestamp": -1, - "file_sizes": {} + "memory_in_bytes": 13852, + "norms_memory_in_bytes": 256, + "points_memory_in_bytes": 8, + "stored_fields_memory_in_bytes": 2496, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 10412, + "version_map_memory_in_bytes": 0 + }, + "shards": { + "index": { + "primaries": { + "avg": 1, + "max": 1, + "min": 1 + }, + "replication": { + "avg": 0, + "max": 0, + "min": 0 + }, + "shards": { + "avg": 1, + "max": 1, + "min": 1 + } + }, + "primaries": 1, + "replication": 0, + "total": 1 + }, + "store": { + "size_in_bytes": 34095 } }, "nodes": { "count": { - "total": 1, - "data": 1, "coordinating_only": 0, + "data": 1, + "ingest": 1, "master": 1, - "ingest": 1 - }, - "versions": ["7.0.0-alpha1"], - "os": { - "available_processors": 4, - "allocated_processors": 1, - "names": [ - { - "name": "Mac OS X", - "count": 1 - } - ], - "mem": { - "total_in_bytes": 17179869184, - "free_in_bytes": 183799808, - "used_in_bytes": 16996069376, - "free_percent": 1, - "used_percent": 99 - } + "total": 1 }, - "process": { - "cpu": { - "percent": 0 - }, - "open_file_descriptors": { - "min": 163, - "max": 163, - "avg": 163 - } + "fs": { + "available_in_bytes": 200403197952, + "free_in_bytes": 200665341952, + "total_in_bytes": 499065712640 }, "jvm": { "max_uptime_in_millis": 701043, + "mem": { + "heap_max_in_bytes": 628555776, + "heap_used_in_bytes": 204299464 + }, + "threads": 40, "versions": [ { - "vm_version": "25.121-b13", "count": 1, - "vm_vendor": "Oracle Corporation", "version": "1.8.0_121", - "vm_name": "Java HotSpot(TM) 64-Bit Server VM" + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "vm_vendor": "Oracle Corporation", + "vm_version": "25.121-b13" } - ], - "mem": { - "heap_used_in_bytes": 204299464, - "heap_max_in_bytes": 628555776 + ] + }, + "network_types": { + "http_types": { + "security4": 1 }, - "threads": 40 + "transport_types": { + "security4": 1 + } }, - "fs": { - "total_in_bytes": 499065712640, - "free_in_bytes": 200665341952, - "available_in_bytes": 200403197952 + "os": { + "allocated_processors": 1, + "available_processors": 4, + "mem": { + "free_in_bytes": 183799808, + "free_percent": 1, + "total_in_bytes": 17179869184, + "used_in_bytes": 16996069376, + "used_percent": 99 + }, + "names": [ + { + "count": 1, + "name": "Mac OS X" + } + ] }, "plugins": [ { "classname": "org.elasticsearch.xpack.XPackPlugin", - "name": "x-pack", "description": "Elasticsearch Expanded Pack Plugin", - "version": "7.0.0-alpha1", - "has_native_controller": true + "has_native_controller": true, + "name": "x-pack", + "version": "7.0.0-alpha1" } ], - "network_types": { - "transport_types": { - "security4": 1 + "process": { + "cpu": { + "percent": 0 }, - "http_types": { - "security4": 1 + "open_file_descriptors": { + "avg": 163, + "max": 163, + "min": 163 } - } - } + }, + "versions": [ + "7.0.0-alpha1" + ] + }, + "status": "green", + "timestamp": 1502835059952 }, + "cluster_uuid": "6d-9tDFTRe-qT5GoBytdlQ", "stack_stats": { "xpack": { + "ccr": { + "auto_follow_patterns_count": 0, + "available": true, + "enabled": true, + "follower_indices_count": 0 + }, + "graph": { + "available": false, + "enabled": true + }, + "logstash": { + "available": false, + "enabled": true + }, + "ml": { + "available": false, + "datafeeds": { + "_all": { + "count": 0 + } + }, + "enabled": true, + "jobs": { + "_all": { + "count": 0, + "detectors": { + "avg": 0, + "max": 0, + "min": 0, + "total": 0 + }, + "model_size": { + "avg": 0, + "max": 0, + "min": 0, + "total": 0 + } + } + } + }, + "monitoring": { + "available": true, + "enabled": true, + "enabled_exporters": { + "http": 1 + } + }, "security": { + "anonymous": { + "enabled": false + }, + "audit": { + "enabled": false, + "outputs": [ + "logfile" + ] + }, "available": false, "enabled": true, + "ipfilter": { + "http": false, + "transport": false + }, "realms": { - "file": { + "active_directory": { "available": false, "enabled": false }, - "ldap": { + "file": { "available": false, "enabled": false }, - "native": { + "ldap": { "available": false, "enabled": false }, - "active_directory": { + "native": { "available": false, "enabled": false }, @@ -188,46 +247,28 @@ "enabled": false } }, - "roles": { + "role_mapping": { "native": { - "size": 1, - "dls": false, - "fls": false - }, - "file": { - "size": 0, - "dls": false, - "fls": false + "enabled": 0, + "size": 0 } }, - "role_mapping": { + "roles": { + "file": { + "dls": false, + "fls": false, + "size": 0 + }, "native": { - "size": 0, - "enabled": 0 + "dls": false, + "fls": false, + "size": 1 } }, "ssl": { "http": { "enabled": false } - }, - "audit": { - "outputs": ["logfile"], - "enabled": false - }, - "ipfilter": { - "http": false, - "transport": false - }, - "anonymous": { - "enabled": false - } - }, - "monitoring": { - "available": true, - "enabled": true, - "enabled_exporters": { - "http": 1 } }, "watcher": { @@ -241,286 +282,302 @@ } } } - }, - "graph": { - "available": false, - "enabled": true - }, - "ml": { - "available": false, - "enabled": true, - "jobs": { - "_all": { - "count": 0, - "detectors": { - "total": 0, - "min": 0, - "avg": 0, - "max": 0 - }, - "model_size": { - "total": 0, - "min": 0, - "avg": 0, - "max": 0 - } - } - }, - "datafeeds": { - "_all": { - "count": 0 - } - } - }, - "logstash": { - "available": false, - "enabled": true - }, - "ccr": { - "auto_follow_patterns_count": 0, - "available": true, - "follower_indices_count": 0, - "enabled": true } } - } + }, + "timestamp": "2017-08-15T22:10:59.952Z", + "version": "7.0.0-alpha1", + "collectionSource": "monitoring" }, { - "cluster_uuid": "lOF8kofiS_2DX58o9mXJ1Q", - "collectionSource": "monitoring", - "timestamp": "2017-08-15T22:10:54.610Z", - "cluster_name": "monitoring-one", - "version": "7.0.0-alpha1", "license": { - "status": "active", - "type": "trial", - "issue_date": "2017-08-15T21:58:28.997Z", "expiry_date": "2017-09-14T21:58:28.997Z", "expiry_date_in_millis": 1505426308997, + "hkey": null, + "issue_date": "2017-08-15T21:58:28.997Z", "issue_date_in_millis": 1502834308997, "issued_to": "monitoring-one", "issuer": "elasticsearch", "max_nodes": 1000, "start_date_in_millis": -1, - "uid": "e5f6e897-0db5-4042-ad7c-6628ddc91691", - "hkey": null + "status": "active", + "type": "trial", + "uid": "e5f6e897-0db5-4042-ad7c-6628ddc91691" }, + "cluster_name": "monitoring-one", "cluster_stats": { - "timestamp": 1502835054610, - "status": "yellow", "indices": { - "count": 8, - "shards": { - "total": 8, - "primaries": 8, - "replication": 0, - "index": { - "shards": { - "min": 1, - "max": 1, - "avg": 1 - }, - "primaries": { - "min": 1, - "max": 1, - "avg": 1 - }, - "replication": { - "min": 0, - "max": 0, - "avg": 0 - } - } + "completion": { + "size_in_bytes": 0 }, + "count": 8, "docs": { "count": 3997, "deleted": 69 }, - "store": { - "size_in_bytes": 2647163 - }, "fielddata": { - "memory_size_in_bytes": 2104, - "evictions": 0 + "evictions": 0, + "memory_size_in_bytes": 2104 }, "query_cache": { - "memory_size_in_bytes": 0, - "total_count": 0, + "cache_count": 0, + "cache_size": 0, + "evictions": 0, "hit_count": 0, + "memory_size_in_bytes": 0, "miss_count": 0, - "cache_size": 0, - "cache_count": 0, - "evictions": 0 - }, - "completion": { - "size_in_bytes": 0 + "total_count": 0 }, "segments": { "count": 36, - "memory_in_bytes": 278961, - "terms_memory_in_bytes": 166031, - "stored_fields_memory_in_bytes": 11544, - "term_vectors_memory_in_bytes": 0, - "norms_memory_in_bytes": 6784, - "points_memory_in_bytes": 3250, "doc_values_memory_in_bytes": 91352, - "index_writer_memory_in_bytes": 205347, - "version_map_memory_in_bytes": 26362, + "file_sizes": {}, "fixed_bit_set_memory_in_bytes": 992, + "index_writer_memory_in_bytes": 205347, "max_unsafe_auto_id_timestamp": -1, - "file_sizes": {} + "memory_in_bytes": 278961, + "norms_memory_in_bytes": 6784, + "points_memory_in_bytes": 3250, + "stored_fields_memory_in_bytes": 11544, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 166031, + "version_map_memory_in_bytes": 26362 + }, + "shards": { + "index": { + "primaries": { + "avg": 1, + "max": 1, + "min": 1 + }, + "replication": { + "avg": 0, + "max": 0, + "min": 0 + }, + "shards": { + "avg": 1, + "max": 1, + "min": 1 + } + }, + "primaries": 8, + "replication": 0, + "total": 8 + }, + "store": { + "size_in_bytes": 2647163 } }, "nodes": { "count": { - "total": 1, - "data": 1, "coordinating_only": 0, + "data": 1, + "ingest": 1, "master": 1, - "ingest": 1 + "total": 1 }, - "versions": ["7.0.0-alpha1"], - "os": { - "available_processors": 4, - "allocated_processors": 1, - "names": [ - { - "name": "Mac OS X", - "count": 1 - } - ], - "mem": { - "total_in_bytes": 17179869184, - "free_in_bytes": 86732800, - "used_in_bytes": 17093136384, - "free_percent": 1, - "used_percent": 99 - } - }, - "process": { - "cpu": { - "percent": 2 - }, - "open_file_descriptors": { - "min": 178, - "max": 178, - "avg": 178 - } + "fs": { + "available_in_bytes": 200403648512, + "free_in_bytes": 200665792512, + "total_in_bytes": 499065712640 }, "jvm": { "max_uptime_in_millis": 761002, + "mem": { + "heap_max_in_bytes": 628555776, + "heap_used_in_bytes": 133041176 + }, + "threads": 42, "versions": [ { - "vm_version": "25.121-b13", "count": 1, - "vm_vendor": "Oracle Corporation", "version": "1.8.0_121", - "vm_name": "Java HotSpot(TM) 64-Bit Server VM" + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "vm_vendor": "Oracle Corporation", + "vm_version": "25.121-b13" } - ], - "mem": { - "heap_used_in_bytes": 133041176, - "heap_max_in_bytes": 628555776 + ] + }, + "network_types": { + "http_types": { + "security4": 1 }, - "threads": 42 + "transport_types": { + "security4": 1 + } }, - "fs": { - "total_in_bytes": 499065712640, - "free_in_bytes": 200665792512, - "available_in_bytes": 200403648512 + "os": { + "allocated_processors": 1, + "available_processors": 4, + "mem": { + "free_in_bytes": 86732800, + "free_percent": 1, + "total_in_bytes": 17179869184, + "used_in_bytes": 17093136384, + "used_percent": 99 + }, + "names": [ + { + "count": 1, + "name": "Mac OS X" + } + ] }, "plugins": [ { "classname": "org.elasticsearch.xpack.XPackPlugin", - "name": "x-pack", "description": "Elasticsearch Expanded Pack Plugin", - "version": "7.0.0-alpha1", - "has_native_controller": true + "has_native_controller": true, + "name": "x-pack", + "version": "7.0.0-alpha1" } ], - "network_types": { - "transport_types": { - "security4": 1 + "process": { + "cpu": { + "percent": 2 }, - "http_types": { - "security4": 1 + "open_file_descriptors": { + "avg": 178, + "max": 178, + "min": 178 } - } - } + }, + "versions": [ + "7.0.0-alpha1" + ] + }, + "status": "yellow", + "timestamp": 1502835054610 }, + "cluster_uuid": "lOF8kofiS_2DX58o9mXJ1Q", "stack_stats": { "xpack": { + "ccr": { + "auto_follow_patterns_count": 0, + "available": true, + "enabled": true, + "follower_indices_count": 0 + }, + "graph": { + "available": true, + "enabled": true + }, + "logstash": { + "available": true, + "enabled": true + }, + "ml": { + "available": true, + "datafeeds": { + "_all": { + "count": 0 + } + }, + "enabled": true, + "jobs": { + "_all": { + "count": 0, + "detectors": { + "avg": 0, + "max": 0, + "min": 0, + "total": 0 + }, + "model_size": { + "avg": 0, + "max": 0, + "min": 0, + "total": 0 + } + } + } + }, + "monitoring": { + "available": true, + "enabled": true, + "enabled_exporters": { + "local": 1 + } + }, "security": { + "anonymous": { + "enabled": false + }, + "audit": { + "enabled": false, + "outputs": [ + "logfile" + ] + }, "available": true, "enabled": true, + "ipfilter": { + "http": false, + "transport": false + }, "realms": { + "active_directory": { + "available": true, + "enabled": false + }, "file": { - "name": ["default_file"], "available": true, - "size": [0], "enabled": true, - "order": [2147483647] + "name": [ + "default_file" + ], + "order": [ + 2147483647 + ], + "size": [ + 0 + ] }, "ldap": { "available": true, "enabled": false }, "native": { - "name": ["default_native"], "available": true, - "size": [2], "enabled": true, - "order": [2147483647] - }, - "active_directory": { - "available": true, - "enabled": false + "name": [ + "default_native" + ], + "order": [ + 2147483647 + ], + "size": [ + 2 + ] }, "pki": { "available": true, "enabled": false } }, - "roles": { + "role_mapping": { "native": { - "size": 1, - "dls": false, - "fls": false - }, - "file": { - "size": 0, - "dls": false, - "fls": false + "enabled": 0, + "size": 0 } }, - "role_mapping": { + "roles": { + "file": { + "dls": false, + "fls": false, + "size": 0 + }, "native": { - "size": 0, - "enabled": 0 + "dls": false, + "fls": false, + "size": 1 } }, "ssl": { "http": { "enabled": false } - }, - "audit": { - "outputs": ["logfile"], - "enabled": false - }, - "ipfilter": { - "http": false, - "transport": false - }, - "anonymous": { - "enabled": false - } - }, - "monitoring": { - "available": true, - "enabled": true, - "enabled_exporters": { - "local": 1 } }, "watcher": { @@ -528,10 +585,6 @@ "enabled": true, "execution": { "actions": { - "index": { - "total": 14, - "total_time_in_ms": 158 - }, "_all": { "total": 110, "total_time_in_ms": 2245 @@ -539,289 +592,342 @@ "email": { "total": 14, "total_time_in_ms": 3 - } - } - } - }, - "graph": { - "available": true, - "enabled": true - }, - "ml": { - "available": true, - "enabled": true, - "jobs": { - "_all": { - "count": 0, - "detectors": { - "total": 0, - "min": 0, - "avg": 0, - "max": 0 }, - "model_size": { - "total": 0, - "min": 0, - "avg": 0, - "max": 0 + "index": { + "total": 14, + "total_time_in_ms": 158 } } - }, - "datafeeds": { - "_all": { - "count": 0 - } } - }, - "logstash": { - "available": true, - "enabled": true - }, - "ccr": { - "auto_follow_patterns_count": 0, - "available": true, - "follower_indices_count": 0, - "enabled": true } } - } + }, + "timestamp": "2017-08-15T22:10:54.610Z", + "version": "7.0.0-alpha1", + "collectionSource": "monitoring" }, { - "cluster_uuid": "TkHOX_-1TzWwbROwQJU5IA", - "collectionSource": "monitoring", - "timestamp": "2017-08-15T22:10:52.642Z", - "cluster_name": "clusterone", - "version": "7.0.0-alpha1", "license": { - "status": "active", - "type": "trial", - "issue_date": "2017-08-15T21:58:47.135Z", "expiry_date": "2017-09-14T21:58:47.135Z", "expiry_date_in_millis": 1505426327135, + "hkey": null, + "issue_date": "2017-08-15T21:58:47.135Z", "issue_date_in_millis": 1502834327135, "issued_to": "clusterone", "issuer": "elasticsearch", "max_nodes": 1000, "start_date_in_millis": -1, - "uid": "e5e99511-0928-41a3-97b0-ec77fa5e3b4c", - "hkey": null + "status": "active", + "type": "trial", + "uid": "e5e99511-0928-41a3-97b0-ec77fa5e3b4c" }, + "cluster_name": "clusterone", "cluster_stats": { - "timestamp": 1502835052641, - "status": "green", "indices": { - "count": 5, - "shards": { - "total": 26, - "primaries": 13, - "replication": 1, - "index": { - "shards": { - "min": 2, - "max": 10, - "avg": 5.2 - }, - "primaries": { - "min": 1, - "max": 5, - "avg": 2.6 - }, - "replication": { - "min": 1, - "max": 1, - "avg": 1 - } - } + "completion": { + "size_in_bytes": 0 }, + "count": 5, "docs": { "count": 150, "deleted": 0 }, - "store": { - "size_in_bytes": 4838464 - }, "fielddata": { - "memory_size_in_bytes": 0, - "evictions": 0 + "evictions": 0, + "memory_size_in_bytes": 0 }, "query_cache": { - "memory_size_in_bytes": 0, - "total_count": 0, + "cache_count": 0, + "cache_size": 0, + "evictions": 0, "hit_count": 0, + "memory_size_in_bytes": 0, "miss_count": 0, - "cache_size": 0, - "cache_count": 0, - "evictions": 0 - }, - "completion": { - "size_in_bytes": 0 + "total_count": 0 }, "segments": { "count": 76, - "memory_in_bytes": 1907922, - "terms_memory_in_bytes": 1595112, - "stored_fields_memory_in_bytes": 23744, - "term_vectors_memory_in_bytes": 0, - "norms_memory_in_bytes": 197184, - "points_memory_in_bytes": 3818, "doc_values_memory_in_bytes": 88064, - "index_writer_memory_in_bytes": 7006184, - "version_map_memory_in_bytes": 260, + "file_sizes": {}, "fixed_bit_set_memory_in_bytes": 0, + "index_writer_memory_in_bytes": 7006184, "max_unsafe_auto_id_timestamp": 1502834982386, - "file_sizes": {} - } - }, - "nodes": { - "count": { - "total": 2, - "data": 2, - "coordinating_only": 0, - "master": 2, - "ingest": 2 + "memory_in_bytes": 1907922, + "norms_memory_in_bytes": 197184, + "points_memory_in_bytes": 3818, + "stored_fields_memory_in_bytes": 23744, + "term_vectors_memory_in_bytes": 0, + "terms_memory_in_bytes": 1595112, + "version_map_memory_in_bytes": 260 }, - "versions": ["7.0.0-alpha1"], - "os": { - "available_processors": 8, - "allocated_processors": 2, - "names": [ - { - "name": "Mac OS X", - "count": 2 + "shards": { + "index": { + "primaries": { + "avg": 2.6, + "max": 5, + "min": 1 + }, + "replication": { + "avg": 1, + "max": 1, + "min": 1 + }, + "shards": { + "avg": 5.2, + "max": 10, + "min": 2 } - ], - "mem": { - "total_in_bytes": 34359738368, - "free_in_bytes": 332099584, - "used_in_bytes": 34027638784, - "free_percent": 1, - "used_percent": 99 - } + }, + "primaries": 13, + "replication": 1, + "total": 26 + }, + "store": { + "size_in_bytes": 4838464 + } + }, + "nodes": { + "count": { + "coordinating_only": 0, + "data": 2, + "ingest": 2, + "master": 2, + "total": 2 }, - "process": { - "cpu": { - "percent": 2 - }, - "open_file_descriptors": { - "min": 218, - "max": 237, - "avg": 227 - } + "fs": { + "available_in_bytes": 200404209664, + "free_in_bytes": 200666353664, + "total_in_bytes": 499065712640 }, "jvm": { "max_uptime_in_millis": 741786, + "mem": { + "heap_max_in_bytes": 1257111552, + "heap_used_in_bytes": 465621856 + }, + "threads": 92, "versions": [ { - "vm_version": "25.121-b13", "count": 2, - "vm_vendor": "Oracle Corporation", "version": "1.8.0_121", - "vm_name": "Java HotSpot(TM) 64-Bit Server VM" + "vm_name": "Java HotSpot(TM) 64-Bit Server VM", + "vm_vendor": "Oracle Corporation", + "vm_version": "25.121-b13" } - ], - "mem": { - "heap_used_in_bytes": 465621856, - "heap_max_in_bytes": 1257111552 + ] + }, + "network_types": { + "http_types": { + "security4": 2 }, - "threads": 92 + "transport_types": { + "security4": 2 + } }, - "fs": { - "total_in_bytes": 499065712640, - "free_in_bytes": 200666353664, - "available_in_bytes": 200404209664 + "os": { + "allocated_processors": 2, + "available_processors": 8, + "mem": { + "free_in_bytes": 332099584, + "free_percent": 1, + "total_in_bytes": 34359738368, + "used_in_bytes": 34027638784, + "used_percent": 99 + }, + "names": [ + { + "count": 2, + "name": "Mac OS X" + } + ] }, "plugins": [ { "classname": "org.elasticsearch.xpack.XPackPlugin", - "name": "x-pack", "description": "Elasticsearch Expanded Pack Plugin", - "version": "7.0.0-alpha1", - "has_native_controller": true + "has_native_controller": true, + "name": "x-pack", + "version": "7.0.0-alpha1" } ], - "network_types": { - "transport_types": { - "security4": 2 + "process": { + "cpu": { + "percent": 2 }, - "http_types": { - "security4": 2 + "open_file_descriptors": { + "avg": 227, + "max": 237, + "min": 218 } - } - } + }, + "versions": [ + "7.0.0-alpha1" + ] + }, + "status": "green", + "timestamp": 1502835052641 }, + "cluster_uuid": "TkHOX_-1TzWwbROwQJU5IA", "stack_stats": { "xpack": { + "ccr": { + "auto_follow_patterns_count": 0, + "available": true, + "enabled": true, + "follower_indices_count": 0 + }, + "graph": { + "available": true, + "enabled": true, + "graph_workspace": { + "total": 0 + } + }, + "logstash": { + "available": true, + "enabled": true + }, + "ml": { + "available": true, + "datafeeds": { + "_all": { + "count": 0 + } + }, + "enabled": true, + "jobs": { + "_all": { + "count": 3, + "detectors": { + "avg": 1, + "max": 1, + "min": 1, + "total": 3 + }, + "model_size": { + "avg": 0, + "max": 0, + "min": 0, + "total": 0 + } + }, + "closed": { + "count": 2, + "detectors": { + "avg": 1, + "max": 1, + "min": 1, + "total": 2 + }, + "model_size": { + "avg": 0, + "max": 0, + "min": 0, + "total": 0 + } + }, + "opened": { + "count": 1, + "detectors": { + "avg": 1, + "max": 1, + "min": 1, + "total": 1 + }, + "model_size": { + "avg": 0, + "max": 0, + "min": 0, + "total": 0 + } + } + } + }, + "monitoring": { + "available": true, + "enabled": true, + "enabled_exporters": { + "http": 1 + } + }, "security": { + "anonymous": { + "enabled": false + }, + "audit": { + "enabled": false, + "outputs": [ + "logfile" + ] + }, "available": true, "enabled": true, + "ipfilter": { + "http": false, + "transport": false + }, "realms": { + "active_directory": { + "available": true, + "enabled": false + }, "file": { - "name": ["default_file"], "available": true, - "size": [0], "enabled": true, - "order": [2147483647] + "name": [ + "default_file" + ], + "order": [ + 2147483647 + ], + "size": [ + 0 + ] }, "ldap": { "available": true, "enabled": false }, "native": { - "name": ["default_native"], "available": true, - "size": [1], "enabled": true, - "order": [2147483647] - }, - "active_directory": { - "available": true, - "enabled": false + "name": [ + "default_native" + ], + "order": [ + 2147483647 + ], + "size": [ + 1 + ] }, "pki": { "available": true, "enabled": false } }, - "roles": { + "role_mapping": { "native": { - "size": 1, - "dls": false, - "fls": false - }, - "file": { - "size": 0, - "dls": false, - "fls": false + "enabled": 0, + "size": 0 } }, - "role_mapping": { + "roles": { + "file": { + "dls": false, + "fls": false, + "size": 0 + }, "native": { - "size": 0, - "enabled": 0 + "dls": false, + "fls": false, + "size": 1 } }, "ssl": { "http": { "enabled": false } - }, - "audit": { - "outputs": ["logfile"], - "enabled": false - }, - "ipfilter": { - "http": false, - "transport": false - }, - "anonymous": { - "enabled": false - } - }, - "monitoring": { - "available": true, - "enabled": true, - "enabled_exporters": { - "http": 1 } }, "watcher": { @@ -835,79 +941,6 @@ } } } - }, - "graph": { - "available": true, - "enabled": true, - "graph_workspace": { - "total": 0 - } - }, - "ml": { - "available": true, - "enabled": true, - "jobs": { - "_all": { - "count": 3, - "detectors": { - "total": 3, - "min": 1, - "avg": 1, - "max": 1 - }, - "model_size": { - "total": 0, - "min": 0, - "avg": 0, - "max": 0 - } - }, - "opened": { - "count": 1, - "detectors": { - "total": 1, - "min": 1, - "avg": 1, - "max": 1 - }, - "model_size": { - "total": 0, - "min": 0, - "avg": 0, - "max": 0 - } - }, - "closed": { - "count": 2, - "detectors": { - "total": 2, - "min": 1, - "avg": 1, - "max": 1 - }, - "model_size": { - "total": 0, - "min": 0, - "avg": 0, - "max": 0 - } - } - }, - "datafeeds": { - "_all": { - "count": 0 - } - } - }, - "logstash": { - "available": true, - "enabled": true - }, - "ccr": { - "auto_follow_patterns_count": 0, - "available": true, - "follower_indices_count": 0, - "enabled": true } }, "kibana": { @@ -944,22 +977,10 @@ }, "indices": 1, "plugins": {} - }, - "logstash": { - "count": 1, - "versions": [ - { - "version": "7.0.0-alpha1", - "count": 1 - } - ], - "os": { - "platforms": [], - "platformReleases": [], - "distros": [], - "distroReleases": [] - } } - } + }, + "timestamp": "2017-08-15T22:10:52.642Z", + "version": "7.0.0-alpha1", + "collectionSource": "monitoring" } ] diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry.js b/x-pack/test/api_integration/apis/telemetry/telemetry.js index 24a8be3e44914d..b21ca27167bd03 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry.js +++ b/x-pack/test/api_integration/apis/telemetry/telemetry.js @@ -15,18 +15,17 @@ export default function ({ getService }) { describe('/api/telemetry/v2/clusters/_stats', () => { it('should load multiple trial-license clusters', async () => { const archive = 'monitoring/multicluster'; - const timeRange = { - min: '2017-08-15T21:00:00Z', - max: '2017-08-16T00:00:00Z', - }; + const timestamp = '2017-08-16T00:00:00Z'; await esArchiver.load(archive); const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); + + expect(body).length(3); expect(body).to.eql(multiClusterFixture); await esArchiver.unload(archive); @@ -35,18 +34,16 @@ export default function ({ getService }) { describe('with basic cluster and reporting and canvas usage info', () => { it('should load non-expiring basic cluster', async () => { const archive = 'monitoring/basic_6.3.x'; - const timeRange = { - min: '2018-07-23T22:07:00Z', - max: '2018-07-23T22:13:00Z', - }; + const timestamp = '2018-07-23T22:13:00Z'; await esArchiver.load(archive); const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); + expect(body).to.eql(basicClusterFixture); await esArchiver.unload(archive); diff --git a/x-pack/test/api_integration/apis/telemetry/telemetry_local.js b/x-pack/test/api_integration/apis/telemetry/telemetry_local.js index 9dbc3e1c8a5bb9..a186e6561e4f94 100644 --- a/x-pack/test/api_integration/apis/telemetry/telemetry_local.js +++ b/x-pack/test/api_integration/apis/telemetry/telemetry_local.js @@ -45,15 +45,12 @@ export default function ({ getService }) { }); it('should pull local stats and validate data types', async () => { - const timeRange = { - min: '2018-07-23T22:07:00Z', - max: '2018-07-23T22:13:00Z', - }; + const timestamp = '2018-07-23T22:13:00Z'; const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); expect(body.length).to.be(1); @@ -105,15 +102,12 @@ export default function ({ getService }) { }); it('should pull local stats and validate fields', async () => { - const timeRange = { - min: '2018-07-23T22:07:00Z', - max: '2018-07-23T22:13:00Z', - }; + const timestamp = '2018-07-23T22:13:00Z'; const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') .set('kbn-xsrf', 'xxx') - .send({ timeRange, unencrypted: true }) + .send({ timestamp, unencrypted: true }) .expect(200); const stats = body[0]; diff --git a/x-pack/test/api_integration/services/usage_api.ts b/x-pack/test/api_integration/services/usage_api.ts index ef6f10f2d4f463..c56de5127f743c 100644 --- a/x-pack/test/api_integration/services/usage_api.ts +++ b/x-pack/test/api_integration/services/usage_api.ts @@ -39,7 +39,7 @@ export function UsageAPIProvider({ getService }: FtrProviderContext) { */ async getTelemetryStats(payload: { unencrypted?: boolean; - timeRange: { min: Date; max: Date }; + timestamp: number | string; }): Promise { const { body } = await supertest .post('/api/telemetry/v2/clusters/_stats') diff --git a/x-pack/test/functional/apps/infra/home_page.ts b/x-pack/test/functional/apps/infra/home_page.ts index 82aba0503fb9d0..7637e573e8e393 100644 --- a/x-pack/test/functional/apps/infra/home_page.ts +++ b/x-pack/test/functional/apps/infra/home_page.ts @@ -64,10 +64,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { .set(COMMON_REQUEST_HEADERS) .set('Accept', 'application/json') .send({ - timeRange: { - min: moment().subtract(1, 'hour').toISOString(), - max: moment().toISOString(), - }, + timestamp: moment().toISOString(), unencrypted: true, }) .expect(200) @@ -88,10 +85,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { .set(COMMON_REQUEST_HEADERS) .set('Accept', 'application/json') .send({ - timeRange: { - min: moment().subtract(1, 'hour').toISOString(), - max: moment().toISOString(), - }, + timestamp: moment().toISOString(), unencrypted: true, }) .expect(200) diff --git a/x-pack/test/functional/apps/infra/logs_source_configuration.ts b/x-pack/test/functional/apps/infra/logs_source_configuration.ts index 04ffcc4847d54f..b0ef147b40f2e0 100644 --- a/x-pack/test/functional/apps/infra/logs_source_configuration.ts +++ b/x-pack/test/functional/apps/infra/logs_source_configuration.ts @@ -118,10 +118,7 @@ export default ({ getPageObjects, getService }: FtrProviderContext) => { .set(COMMON_REQUEST_HEADERS) .set('Accept', 'application/json') .send({ - timeRange: { - min: moment().subtract(1, 'hour').toISOString(), - max: moment().toISOString(), - }, + timestamp: moment().toISOString(), unencrypted: true, }) .expect(200)