Add log rate spike analysis to Logs alert details page (#158960)

Resolves elastic/actionable-observability#8

Adds log rate spike analysis to Logs threshold alert details page for
the "Count" alerts.

The logs spike analysis is shown for the following configurations:
- Count alerts when condition is to check "more than" or "more than or
equals" threshold

The logs spike analysis is hidden for the following configurations:
- Ratio alerts
- If rule params contain group by
- If the condition is to check "less than" or "less than or equals"
threshold

### Note about Co-Pilot
The PR also integrates Co-Pilot prompt. The prompt message needs to be
updated with the correct field values from the analysis results, which
will be done in a separate PR.

### Manual testing
1. Create a Log threshold rule
2. Ensure "Count" is selected
3. Ensure "more than" or "more than or equals" is selected for the
threshold
4. Enter threshold
5. Save rule
6. Wait for alerts to be generated
7. Go to alert details page
8. Ensure "Explain Log Rate Spikes" is shown

[Guide to generate test
data](https://docs.google.com/document/d/1PfPU5mjFx_jWFUkgtaHVy7qPvgBC7g1l4LKe7Jn-qUc)

### Feature flag
The Logs threshold alert details page is behind a feature flag.

To test, add the following in the `kibana.dev.yml`:
 `xpack.observability.unsafe.alertDetails.logs.enabled: true`


<img width="1511" alt="Screenshot 2023-06-12 at 20 18 34"
src="https://github.com/elastic/kibana/assets/69037875/dd9391ad-53cd-4484-a11c-090f6a9acdd1">

---------

Co-authored-by: kibanamachine <42973632+kibanamachine@users.noreply.github.com>

packages/kbn-optimizer/limits.yml

@@ -95,7 +95,7 @@ pageLoadAssetSize:
   monitoring: 80000
   navigation: 37269
   newsfeed: 42228
-  observability: 100000
+  observability: 105500
   observabilityOnboarding: 19573
   observabilityShared: 52256
   osquery: 107090

x-pack/plugins/infra/common/alerting/logs/log_threshold/query_helpers.ts

@@ -0,0 +1,165 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import { RuleParams, Comparator, CountCriteria, Criterion, ExecutionTimeRange } from '.';
+
+import { getIntervalInSeconds } from '../../../utils/get_interval_in_seconds';
+
+export type LogThresholdRuleTypeParams = RuleParams;
+
+export const buildFiltersFromCriteria = (
+  params: Pick<RuleParams, 'timeSize' | 'timeUnit'> & { criteria: CountCriteria },
+  timestampField: string,
+  executionTimeRange?: ExecutionTimeRange
+) => {
+  const { timeSize, timeUnit, criteria } = params;
+  const interval = `${timeSize}${timeUnit}`;
+  const intervalAsSeconds = getIntervalInSeconds(interval);
+  const intervalAsMs = intervalAsSeconds * 1000;
+  const to = executionTimeRange?.lte || Date.now();
+  const from = executionTimeRange?.gte || to - intervalAsMs;
+
+  const positiveCriteria = criteria.filter((criterion) =>
+    positiveComparators.includes(criterion.comparator)
+  );
+  const negativeCriteria = criteria.filter((criterion) =>
+    negativeComparators.includes(criterion.comparator)
+  );
+  // Positive assertions (things that "must" match)
+  const mustFilters = buildFiltersForCriteria(positiveCriteria);
+  // Negative assertions (things that "must not" match)
+  const mustNotFilters = buildFiltersForCriteria(negativeCriteria);
+
+  const rangeFilter = {
+    range: {
+      [timestampField]: {
+        gte: from,
+        lte: to,
+        format: 'epoch_millis',
+      },
+    },
+  };
+
+  // For group by scenarios we'll pad the time range by 1 x the interval size on the left (lte) and right (gte), this is so
+  // a wider net is cast to "capture" the groups. This is to account for scenarios where we want ascertain if
+  // there were "no documents" (less than 1 for example). In these cases we may be missing documents to build the groups
+  // and match / not match the criteria.
+  const groupedRangeFilter = {
+    range: {
+      [timestampField]: {
+        gte: from - intervalAsMs,
+        lte: to + intervalAsMs,
+        format: 'epoch_millis',
+      },
+    },
+  };
+
+  return { rangeFilter, groupedRangeFilter, mustFilters, mustNotFilters };
+};
+
+const buildFiltersForCriteria = (criteria: CountCriteria) => {
+  let filters: estypes.QueryDslQueryContainer[] = [];
+
+  criteria.forEach((criterion) => {
+    const criterionQuery = buildCriterionQuery(criterion);
+    if (criterionQuery) {
+      filters = [...filters, criterionQuery];
+    }
+  });
+  return filters;
+};
+
+const buildCriterionQuery = (criterion: Criterion): estypes.QueryDslQueryContainer | undefined => {
+  const { field, value, comparator } = criterion;
+
+  const queryType = getQueryMappingForComparator(comparator);
+
+  switch (queryType) {
+    case 'term':
+      return {
+        term: {
+          [field]: {
+            value,
+          },
+        },
+      };
+    case 'match': {
+      return {
+        match: {
+          [field]: value,
+        },
+      };
+    }
+    case 'match_phrase': {
+      return {
+        match_phrase: {
+          [field]: String(value),
+        },
+      };
+    }
+    case 'range': {
+      const comparatorToRangePropertyMapping: {
+        [key: string]: string;
+      } = {
+        [Comparator.LT]: 'lt',
+        [Comparator.LT_OR_EQ]: 'lte',
+        [Comparator.GT]: 'gt',
+        [Comparator.GT_OR_EQ]: 'gte',
+      };
+
+      const rangeProperty = comparatorToRangePropertyMapping[comparator];
+
+      return {
+        range: {
+          [field]: {
+            [rangeProperty]: value,
+          },
+        },
+      };
+    }
+    default: {
+      return undefined;
+    }
+  }
+};
+
+export const positiveComparators = [
+  Comparator.GT,
+  Comparator.GT_OR_EQ,
+  Comparator.LT,
+  Comparator.LT_OR_EQ,
+  Comparator.EQ,
+  Comparator.MATCH,
+  Comparator.MATCH_PHRASE,
+];
+
+export const negativeComparators = [
+  Comparator.NOT_EQ,
+  Comparator.NOT_MATCH,
+  Comparator.NOT_MATCH_PHRASE,
+];
+
+export const queryMappings: {
+  [key: string]: string;
+} = {
+  [Comparator.GT]: 'range',
+  [Comparator.GT_OR_EQ]: 'range',
+  [Comparator.LT]: 'range',
+  [Comparator.LT_OR_EQ]: 'range',
+  [Comparator.EQ]: 'term',
+  [Comparator.MATCH]: 'match',
+  [Comparator.MATCH_PHRASE]: 'match_phrase',
+  [Comparator.NOT_EQ]: 'term',
+  [Comparator.NOT_MATCH]: 'match',
+  [Comparator.NOT_MATCH_PHRASE]: 'match_phrase',
+};
+
+const getQueryMappingForComparator = (comparator: Comparator) => {
+  return queryMappings[comparator];
+};

x-pack/plugins/infra/kibana.jsonc

@@ -9,6 +9,7 @@
     "browser": true,
     "configPath": ["xpack", "infra"],
     "requiredPlugins": [
+      "aiops",
       "alerting",
       "cases",
       "charts",
@@ -17,6 +18,7 @@
       "discover",
       "embeddable",
       "features",
+      "fieldFormats",
       "lens",
       "observability",
       "observabilityShared",

x-pack/plugins/infra/public/alerting/log_threshold/components/alert_details_app_section/components/explain_log_rate_spike.tsx

@@ -0,0 +1,167 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import React, { FC, useEffect, useState } from 'react';
+import { pick } from 'lodash';
+import moment from 'moment';
+
+import { EuiFlexGroup, EuiFlexItem, EuiPanel, EuiTitle } from '@elastic/eui';
+
+import { FormattedMessage } from '@kbn/i18n-react';
+import { DataView } from '@kbn/data-views-plugin/common';
+import { ExplainLogRateSpikesContent } from '@kbn/aiops-plugin/public';
+
+import { Rule } from '@kbn/alerting-plugin/common';
+import { CoPilotPrompt, TopAlert, useCoPilot } from '@kbn/observability-plugin/public';
+import { QueryDslQueryContainer } from '@elastic/elasticsearch/lib/api/types';
+import { i18n } from '@kbn/i18n';
+import { CoPilotPromptId } from '@kbn/observability-plugin/common';
+import { ALERT_END } from '@kbn/rule-data-utils';
+import { useKibanaContextForPlugin } from '../../../../../hooks/use_kibana';
+import {
+  Comparator,
+  CountRuleParams,
+  hasGroupBy,
+  isRatioRuleParams,
+  PartialRuleParams,
+  ruleParamsRT,
+} from '../../../../../../common/alerting/logs/log_threshold';
+import { decodeOrThrow } from '../../../../../../common/runtime_types';
+import { getESQueryForLogSpike } from '../log_rate_spike_query';
+
+export interface AlertDetailsExplainLogRateSpikesSectionProps {
+  rule: Rule<PartialRuleParams>;
+  alert: TopAlert<Record<string, any>>;
+}
+
+export const ExplainLogRateSpikes: FC<AlertDetailsExplainLogRateSpikesSectionProps> = ({
+  rule,
+  alert,
+}) => {
+  const { services } = useKibanaContextForPlugin();
+  const { dataViews, logViews } = services;
+  const [dataView, setDataView] = useState<DataView | undefined>();
+  const [esSearchQuery, setEsSearchQuery] = useState<QueryDslQueryContainer | undefined>();
+
+  useEffect(() => {
+    const getDataView = async () => {
+      const { timestampField, dataViewReference } = await logViews.client.getResolvedLogView(
+        rule.params.logView
+      );
+
+      if (dataViewReference.id) {
+        const logDataView = await dataViews.get(dataViewReference.id);
+        setDataView(logDataView);
+        getQuery(timestampField);
+      }
+    };
+
+    const getQuery = (timestampField: string) => {
+      const esSearchRequest = getESQueryForLogSpike(
+        validatedParams as CountRuleParams,
+        timestampField
+      ) as QueryDslQueryContainer;
+
+      if (esSearchRequest) {
+        setEsSearchQuery(esSearchRequest);
+      }
+    };
+
+    const validatedParams = decodeOrThrow(ruleParamsRT)(rule.params);
+
+    if (
+      !isRatioRuleParams(validatedParams) &&
+      !hasGroupBy(validatedParams) &&
+      (validatedParams.count.comparator === Comparator.GT ||
+        validatedParams.count.comparator === Comparator.GT_OR_EQ)
+    ) {
+      getDataView();
+    }
+  }, [rule, alert, dataViews, logViews]);
+
+  const alertStart = moment(alert.start);
+  const alertEnd = alert.fields[ALERT_END] ? moment(alert.fields[ALERT_END]) : undefined;
+
+  const timeRange = {
+    min: alertStart.clone().subtract(20, 'minutes'),
+    max: alertEnd ? alertEnd.clone().add(5, 'minutes') : moment(new Date()),
+  };
+
+  const initialAnalysisStart = {
+    baselineMin: alertStart.clone().subtract(10, 'minutes').valueOf(),
+    baselineMax: alertStart.clone().subtract(1, 'minutes').valueOf(),
+    deviationMin: alertStart.valueOf(),
+    deviationMax: alertStart.clone().add(10, 'minutes').isAfter(moment(new Date()))
+      ? moment(new Date()).valueOf()
+      : alertStart.clone().add(10, 'minutes').valueOf(),
+  };
+
+  const coPilotService = useCoPilot();
+
+  const explainLogSpikeParams = undefined;
+
+  const explainLogSpikeTitle = i18n.translate(
+    'xpack.infra.logs.alertDetails.explainLogSpikeTitle',
+    {
+      defaultMessage: 'Possible causes and remediations',
+    }
+  );
+
+  if (!dataView || !esSearchQuery) return null;
+
+  return (
+    <EuiPanel hasBorder={true} data-test-subj="explainLogRateSpikesAlertDetails">
+      <EuiFlexGroup direction="column" gutterSize="none" responsive={false}>
+        <EuiFlexItem grow={false}>
+          <EuiTitle size="xs">
+            <h2>
+              <FormattedMessage
+                id="xpack.infra.logs.alertDetails.explainLogRateSpikes.sectionTitle"
+                defaultMessage="Explain Log Rate Spikes"
+              />
+            </h2>
+          </EuiTitle>
+        </EuiFlexItem>
+        <EuiFlexItem>
+          <ExplainLogRateSpikesContent
+            dataView={dataView}
+            timeRange={timeRange}
+            esSearchQuery={esSearchQuery}
+            initialAnalysisStart={initialAnalysisStart}
+            appDependencies={pick(services, [
+              'application',
+              'data',
+              'executionContext',
+              'charts',
+              'fieldFormats',
+              'http',
+              'notifications',
+              'share',
+              'storage',
+              'uiSettings',
+              'unifiedSearch',
+              'theme',
+              'lens',
+            ])}
+          />
+        </EuiFlexItem>
+      </EuiFlexGroup>
+      <EuiFlexGroup direction="column" gutterSize="m">
+        {coPilotService?.isEnabled() && explainLogSpikeParams ? (
+          <EuiFlexItem grow={false}>
+            <CoPilotPrompt
+              coPilot={coPilotService}
+              title={explainLogSpikeTitle}
+              params={explainLogSpikeParams}
+              promptId={CoPilotPromptId.ExplainLogSpike}
+            />
+          </EuiFlexItem>
+        ) : null}
+      </EuiFlexGroup>
+    </EuiPanel>
+  );
+};