Merge remote-tracking branch 'upstream/7.14' into backport/7.14/pr-10…

…7674

# Conflicts:
#	x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_alert.ts

package.json

@@ -98,7 +98,7 @@
     "@elastic/apm-rum-react": "^1.2.11",
     "@elastic/charts": "32.0.1",
     "@elastic/datemath": "link:bazel-bin/packages/elastic-datemath",
-    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@7.14.0-canary.6",
+    "@elastic/elasticsearch": "npm:@elastic/elasticsearch-canary@7.14.0-canary.7",
     "@elastic/ems-client": "7.14.0",
     "@elastic/eui": "34.5.2",
     "@elastic/filesaver": "1.1.2",

packages/kbn-es/src/utils/native_realm.js

@@ -11,6 +11,23 @@ const chalk = require('chalk');
 
 const { log: defaultLog } = require('./log');
 
+/**
+ * Hack to skip the Product Check performed by the Elasticsearch-js client.
+ * We noticed a couple of bugs that may need to be fixed before taking full
+ * advantage of this feature.
+ *
+ * The bugs are detailed in this issue: https://github.com/elastic/kibana/issues/105557
+ *
+ * The hack is copied from the test/utils in the elasticsearch-js repo
+ * (https://github.com/elastic/elasticsearch-js/blob/master/test/utils/index.js#L45-L56)
+ */
+function skipProductCheck(client) {
+  const tSymbol = Object.getOwnPropertySymbols(client.transport || client).filter(
+    (symbol) => symbol.description === 'product check'
+  )[0];
+  (client.transport || client)[tSymbol] = 2;
+}
+
 exports.NativeRealm = class NativeRealm {
   constructor({ elasticPassword, port, log = defaultLog, ssl = false, caCert }) {
     this._client = new Client({
@@ -22,6 +39,8 @@ exports.NativeRealm = class NativeRealm {
           }
         : undefined,
     });
+    // TODO: @elastic/es-clients I had to disable the product check here because the client is getting 404 while ES is initializing, but the requests here auto retry them.
+    skipProductCheck(this._client);
     this._elasticPassword = elasticPassword;
     this._log = log;
   }

src/core/server/elasticsearch/client/configure_client.ts

@@ -22,6 +22,12 @@ export const configureClient = (
   const client = new Client(clientOptions);
   addLogging(client, logger.get('query', type));
 
+  // ------------------------------------------------------------------------ //
+  // Hack to disable the "Product check" while the bugs in                    //
+  // https://github.com/elastic/kibana/issues/105557 are handled.             //
+  skipProductCheck(client);
+  // ------------------------------------------------------------------------ //
+
   return client;
 };
 
@@ -78,3 +84,21 @@ const addLogging = (client: Client, logger: Logger) => {
     }
   });
 };
+
+/**
+ * Hack to skip the Product Check performed by the Elasticsearch-js client.
+ * We noticed a couple of bugs that may need to be fixed before taking full
+ * advantage of this feature.
+ *
+ * The bugs are detailed in this issue: https://github.com/elastic/kibana/issues/105557
+ *
+ * The hack is copied from the test/utils in the elasticsearch-js repo
+ * (https://github.com/elastic/elasticsearch-js/blob/master/test/utils/index.js#L45-L56)
+ */
+function skipProductCheck(client: Client) {
+  const tSymbol = Object.getOwnPropertySymbols(client.transport || client).filter(
+    (symbol) => symbol.description === 'product check'
+  )[0];
+  // @ts-expect-error `tSymbol` is missing in the index signature of Transport
+  (client.transport || client)[tSymbol] = 2;
+}

src/setup_node_env/exit_on_warning.js

@@ -29,6 +29,14 @@ var IGNORE_WARNINGS = [
     file: '/node_modules/supertest/node_modules/superagent/lib/node/index.js',
     line: 418,
   },
+  {
+    // TODO: @elastic/es-clients - The new client will attempt a Product check and it will `process.emitWarning`
+    //  that the security features are blocking such check.
+    //  Such emit is causing Node.js to crash unless we explicitly catch it.
+    //  We need to discard that warning
+    message:
+      'The client is unable to verify that the server is Elasticsearch due to security privileges on the server side. Some functionality may not be compatible if the server is running an unsupported product.',
+  },
 ];
 
 if (process.noProcessWarnings !== true) {

x-pack/plugins/infra/common/http_api/metrics_api.ts

@@ -35,7 +35,7 @@ export const MetricsAPIRequestRT = rt.intersection([
     afterKey: rt.union([rt.null, afterKeyObjectRT]),
     limit: rt.union([rt.number, rt.null, rt.undefined]),
     filters: rt.array(rt.object),
-    dropLastBucket: rt.boolean,
+    dropPartialBuckets: rt.boolean,
     alignDataToEnd: rt.boolean,
   }),
 ]);

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/evaluate_alert.ts

@@ -96,6 +96,8 @@ export const evaluateAlert = <Params extends EvaluatedAlertParams = EvaluatedAle
   );
 };
 
+const MINIMUM_BUCKETS = 5;
+
 const getMetric: (
   esClient: ElasticsearchClient,
   params: MetricExpressionParams,

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.test.ts

@@ -7,6 +7,7 @@
 
 import { MetricExpressionParams } from '../types';
 import { getElasticsearchMetricQuery } from './metric_query';
+import moment from 'moment';
 
 describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
   const expressionParams = {
@@ -18,9 +19,13 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
 
   const timefield = '@timestamp';
   const groupBy = 'host.doggoname';
+  const timeframe = {
+    start: moment().subtract(5, 'minutes').valueOf(),
+    end: moment().valueOf(),
+  };
 
   describe('when passed no filterQuery', () => {
-    const searchBody = getElasticsearchMetricQuery(expressionParams, timefield, groupBy);
+    const searchBody = getElasticsearchMetricQuery(expressionParams, timefield, timeframe, groupBy);
     test('includes a range filter', () => {
       expect(
         searchBody.query.bool.filter.find((filter) => filter.hasOwnProperty('range'))
@@ -43,6 +48,7 @@ describe("The Metric Threshold Alert's getElasticsearchMetricQuery", () => {
     const searchBody = getElasticsearchMetricQuery(
       expressionParams,
       timefield,
+      timeframe,
       groupBy,
       filterQuery
     );

x-pack/plugins/infra/server/lib/alerting/metric_threshold/lib/metric_query.ts

@@ -4,15 +4,13 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-import moment from 'moment';
+
 import { networkTraffic } from '../../../../../common/inventory_models/shared/metrics/snapshot/network_traffic';
 import { MetricExpressionParams, Aggregators } from '../types';
 import { getIntervalInSeconds } from '../../../../utils/get_interval_in_seconds';
-import { roundTimestamp } from '../../../../utils/round_timestamp';
 import { createPercentileAggregation } from './create_percentile_aggregation';
 import { calculateDateHistogramOffset } from '../../../metrics/lib/calculate_date_histogram_offset';
 
-const MINIMUM_BUCKETS = 5;
 const COMPOSITE_RESULTS_PER_PAGE = 100;
 
 const getParsedFilterQuery: (filterQuery: string | undefined) => Record<string, any> | null = (
@@ -25,9 +23,9 @@ const getParsedFilterQuery: (filterQuery: string | undefined) => Record<string,
 export const getElasticsearchMetricQuery = (
   { metric, aggType, timeUnit, timeSize }: MetricExpressionParams,
   timefield: string,
+  timeframe: { start: number; end: number },
   groupBy?: string | string[],
-  filterQuery?: string,
-  timeframe?: { start: number; end: number }
+  filterQuery?: string
 ) => {
   if (aggType === Aggregators.COUNT && metric) {
     throw new Error('Cannot aggregate document count with a metric');
@@ -38,20 +36,8 @@ export const getElasticsearchMetricQuery = (
   const interval = `${timeSize}${timeUnit}`;
   const intervalAsSeconds = getIntervalInSeconds(interval);
   const intervalAsMS = intervalAsSeconds * 1000;
-
-  const to = moment(timeframe ? timeframe.end : Date.now())
-    .add(1, timeUnit)
-    .startOf(timeUnit)
-    .valueOf();
-
-  // We need enough data for 5 buckets worth of data. We also need
-  // to convert the intervalAsSeconds to milliseconds.
-  const minimumFrom = to - intervalAsMS * MINIMUM_BUCKETS;
-
-  const from = roundTimestamp(
-    timeframe && timeframe.start <= minimumFrom ? timeframe.start : minimumFrom,
-    timeUnit
-  );
+  const to = timeframe.end;
+  const from = timeframe.start;
 
   const deliveryDelay = 60 * 1000; // INFO: This allows us to account for any delay ES has in indexing the most recent data.
 

x-pack/plugins/infra/server/lib/alerting/metric_threshold/metric_threshold_executor.test.ts

@@ -119,14 +119,12 @@ describe('The metric threshold alert type', () => {
       expect(mostRecentAction(instanceID)).toBe(undefined);
     });
     test('reports expected values to the action context', async () => {
-      const now = 1577858400000;
       await execute(Comparator.GT, [0.75]);
       const { action } = mostRecentAction(instanceID);
       expect(action.group).toBe('*');
       expect(action.reason).toContain('current value is 1');
       expect(action.reason).toContain('threshold of 0.75');
       expect(action.reason).toContain('test.metric.1');
-      expect(action.timestamp).toBe(new Date(now).toISOString());
     });
   });
 
@@ -428,15 +426,13 @@ describe('The metric threshold alert type', () => {
         },
       });
     test('reports values converted from decimals to percentages to the action context', async () => {
-      const now = 1577858400000;
       await execute();
       const { action } = mostRecentAction(instanceID);
       expect(action.group).toBe('*');
       expect(action.reason).toContain('current value is 100%');
       expect(action.reason).toContain('threshold of 75%');
       expect(action.threshold.condition0[0]).toBe('75%');
       expect(action.value.condition0).toBe('100%');
-      expect(action.timestamp).toBe(new Date(now).toISOString());
     });
   });
 });
@@ -460,7 +456,8 @@ const executor = createMetricThresholdExecutor(mockLibs);
 
 const services: AlertServicesMock = alertsMock.createAlertServices();
 services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: any): any => {
-  if (params.index === 'alternatebeat-*') return mocks.changedSourceIdResponse;
+  const from = params?.body.query.bool.filter[0]?.range['@timestamp'].gte;
+  if (params.index === 'alternatebeat-*') return mocks.changedSourceIdResponse(from);
   const metric = params?.body.query.bool.filter[1]?.exists.field;
   if (params?.body.aggs.groupings) {
     if (params?.body.aggs.groupings.composite.after) {
@@ -470,25 +467,27 @@ services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: a
     }
     if (metric === 'test.metric.2') {
       return elasticsearchClientMock.createSuccessTransportRequestPromise(
-        mocks.alternateCompositeResponse
+        mocks.alternateCompositeResponse(from)
       );
     }
     return elasticsearchClientMock.createSuccessTransportRequestPromise(
-      mocks.basicCompositeResponse
+      mocks.basicCompositeResponse(from)
     );
   }
   if (metric === 'test.metric.2') {
     return elasticsearchClientMock.createSuccessTransportRequestPromise(
-      mocks.alternateMetricResponse
+      mocks.alternateMetricResponse(from)
     );
   } else if (metric === 'test.metric.3') {
     return elasticsearchClientMock.createSuccessTransportRequestPromise(
-      params?.body.aggs.aggregatedIntervals.aggregations.aggregatedValue_max
+      params?.body.aggs.aggregatedIntervals.aggregations.aggregatedValueMax
         ? mocks.emptyRateResponse
         : mocks.emptyMetricResponse
     );
   }
-  return elasticsearchClientMock.createSuccessTransportRequestPromise(mocks.basicMetricResponse);
+  return elasticsearchClientMock.createSuccessTransportRequestPromise(
+    mocks.basicMetricResponse(from)
+  );
 });
 services.savedObjectsClient.get.mockImplementation(async (type: string, sourceId: string) => {
   if (sourceId === 'alternate')

x-pack/plugins/infra/server/lib/alerting/metric_threshold/preview_metric_threshold_alert.test.ts

@@ -167,6 +167,7 @@ describe('Previewing the metric threshold alert type', () => {
 const services: AlertServicesMock = alertsMock.createAlertServices();
 
 services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: any): any => {
+  const from = params?.body.query.bool.filter[0]?.range['@timestamp'].gte;
   const metric = params?.body.query.bool.filter[1]?.exists.field;
   if (params?.body.aggs.groupings) {
     if (params?.body.aggs.groupings.composite.after) {
@@ -175,21 +176,21 @@ services.scopedClusterClient.asCurrentUser.search.mockImplementation((params?: a
       );
     }
     return elasticsearchClientMock.createSuccessTransportRequestPromise(
-      mocks.basicCompositePreviewResponse
+      mocks.basicCompositePreviewResponse(from)
     );
   }
   if (metric === 'test.metric.2') {
     return elasticsearchClientMock.createSuccessTransportRequestPromise(
-      mocks.alternateMetricPreviewResponse
+      mocks.alternateMetricPreviewResponse(from)
     );
   }
   if (metric === 'test.metric.3') {
     return elasticsearchClientMock.createSuccessTransportRequestPromise(
-      mocks.repeatingMetricPreviewResponse
+      mocks.repeatingMetricPreviewResponse(from)
     );
   }
   return elasticsearchClientMock.createSuccessTransportRequestPromise(
-    mocks.basicMetricPreviewResponse
+    mocks.basicMetricPreviewResponse(from)
   );
 });