[ML] Fix custom index name settings, functional tests for APM Latency…

… Correlation. (#105200) (#106077) Adds first functional tests for APM Latency Correlations code. - Fix: The log log chart's Y axis would only start at 1 by default which hides results for small datasets like the ones used in these tests. Starting the axis at 0 isn't supported to log based ones so we're setting it to just a small value above 0. - Fix: Instead of the hard coded apm-* index we passed on from the client, we now correctly consider APM's custom settings.
elastic · Jul 19, 2021 · faf5baa · faf5baa
1 parent 41416d8
commit faf5baa
Show file tree

Hide file tree

Showing 25 changed files with 409 additions and 88 deletions.
diff --git a/x-pack/plugins/apm/common/search_strategies/correlations/types.ts b/x-pack/plugins/apm/common/search_strategies/correlations/types.ts
@@ -19,7 +19,6 @@ export interface ResponseHit {
 }
 
 export interface SearchServiceParams {
-  index: string;
   environment?: string;
   kuery?: string;
   serviceName?: string;
@@ -31,6 +30,10 @@ export interface SearchServiceParams {
   percentileThresholdValue?: number;
 }
 
+export interface SearchServiceFetchParams extends SearchServiceParams {
+  index: string;
+}
+
 export interface SearchServiceValue {
   histogram: HistogramItem[];
   value: string;

diff --git a/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx b/x-pack/plugins/apm/public/components/app/correlations/correlations_chart.tsx
@@ -70,6 +70,11 @@ const chartTheme: PartialTheme = {
   },
 };
 
+// Log based axis cannot start a 0. Use a small positive number instead.
+const yAxisDomain = {
+  min: 0.00001,
+};
+
 interface CorrelationsChartProps {
   field?: string;
   value?: string;
@@ -140,7 +145,10 @@ export function CorrelationsChart({
   const histogram = replaceHistogramDotsWithBars(originalHistogram);
 
   return (
-    <div style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}>
+    <div
+      data-test-subj="apmCorrelationsChart"
+      style={{ overflow: 'hidden', textOverflow: 'ellipsis' }}
+    >
       <Chart
         size={{
           height: '250px',
@@ -168,6 +176,7 @@ export function CorrelationsChart({
         />
         <Axis
           id="y-axis"
+          domain={yAxisDomain}
           title={i18n.translate(
             'xpack.apm.correlations.latency.chart.numberOfTransactionsLabel',
             { defaultMessage: '# transactions' }

diff --git a/x-pack/plugins/apm/public/components/app/correlations/index.tsx b/x-pack/plugins/apm/public/components/app/correlations/index.tsx
@@ -134,6 +134,7 @@ export function Correlations() {
   return (
     <>
       <EuiButton
+        data-test-subj="apmViewCorrelationsButton"
         fill
         onClick={() => {
           setIsFlyoutVisible(true);
@@ -147,13 +148,17 @@ export function Correlations() {
       {isFlyoutVisible && (
         <EuiPortal>
           <EuiFlyout
+            data-test-subj="apmCorrelationsFlyout"
             size="l"
             ownFocus
             onClose={() => setIsFlyoutVisible(false)}
           >
             <EuiFlyoutHeader hasBorder aria-labelledby="correlations-flyout">
               <EuiTitle>
-                <h2 id="correlations-flyout">
+                <h2
+                  data-test-subj="apmCorrelationsFlyoutHeader"
+                  id="correlations-flyout"
+                >
                   {CORRELATIONS_TITLE}
                   &nbsp;
                   <EuiBetaBadge

diff --git a/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx b/x-pack/plugins/apm/public/components/app/correlations/ml_latency_correlations.tsx
@@ -8,6 +8,10 @@
 import React, { useEffect, useMemo, useState } from 'react';
 import { useHistory } from 'react-router-dom';
 import {
+  EuiCallOut,
+  EuiCode,
+  EuiAccordion,
+  EuiPanel,
   EuiIcon,
   EuiBasicTableColumn,
   EuiButton,
@@ -34,7 +38,10 @@ import {
 } from './correlations_table';
 import { useCorrelations } from './use_correlations';
 import { push } from '../../shared/Links/url_helpers';
-import { useUiTracker } from '../../../../../observability/public';
+import {
+  enableInspectEsQueries,
+  useUiTracker,
+} from '../../../../../observability/public';
 import { asPreciseDecimal } from '../../../../common/utils/formatters';
 import { useApmServiceContext } from '../../../context/apm_service/use_apm_service_context';
 import { LatencyCorrelationsHelpPopover } from './ml_latency_correlations_help_popover';
@@ -58,15 +65,19 @@ interface MlCorrelationsTerms {
 
 export function MlLatencyCorrelations({ onClose }: Props) {
   const {
-    core: { notifications },
+    core: { notifications, uiSettings },
   } = useApmPluginContext();
 
   const { serviceName, transactionType } = useApmServiceContext();
   const { urlParams } = useUrlParams();
 
   const { environment, kuery, transactionName, start, end } = urlParams;
 
+  const displayLog = uiSettings.get<boolean>(enableInspectEsQueries);
+
   const {
+    ccsWarning,
+    log,
     error,
     histograms,
     percentileThresholdValue,
@@ -76,7 +87,6 @@ export function MlLatencyCorrelations({ onClose }: Props) {
     cancelFetch,
     overallHistogram: originalOverallHistogram,
   } = useCorrelations({
-    index: 'apm-*',
     ...{
       ...{
         environment,
@@ -286,9 +296,10 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </EuiFlexItem>
         <EuiFlexItem>
           <EuiFlexGroup direction="column" gutterSize="none">
-            <EuiFlexItem>
+            <EuiFlexItem data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle">
               <EuiText size="xs" color="subdued">
                 <FormattedMessage
+                  data-test-subj="apmCorrelationsLatencyCorrelationsProgressTitle"
                   id="xpack.apm.correlations.latencyCorrelations.progressTitle"
                   defaultMessage="Progress: {progress}%"
                   values={{ progress: Math.round(progress * 100) }}
@@ -313,11 +324,35 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </EuiFlexItem>
       </EuiFlexGroup>
 
+      {ccsWarning && (
+        <>
+          <EuiSpacer size="m" />
+          <EuiCallOut
+            title={i18n.translate(
+              'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutTitle',
+              {
+                defaultMessage: 'Cross-cluster search compatibility',
+              }
+            )}
+            color="warning"
+          >
+            <p>
+              {i18n.translate(
+                'xpack.apm.correlations.latencyCorrelations.ccsWarningCalloutBody',
+                {
+                  defaultMessage:
+                    'Data for the correlation analysis could not be fully retrieved. This feature is supported only for 7.14 and later versions.',
+                }
+              )}
+            </p>
+          </EuiCallOut>
+        </>
+      )}
       <EuiSpacer size="m" />
       {overallHistogram !== undefined ? (
         <>
           <EuiTitle size="xxs">
-            <h4>
+            <h4 data-test-subj="apmCorrelationsLatencyCorrelationsChartTitle">
               {i18n.translate(
                 'xpack.apm.correlations.latencyCorrelations.chartTitle',
                 {
@@ -341,32 +376,58 @@ export function MlLatencyCorrelations({ onClose }: Props) {
         </>
       ) : null}
 
-      {histograms.length > 0 && selectedHistogram !== undefined && (
-        <CorrelationsTable
-          // @ts-ignore correlations don't have the same column format other tables have
-          columns={mlCorrelationColumns}
-          // @ts-expect-error correlations don't have the same significant term other tables have
-          significantTerms={histogramTerms}
-          status={FETCH_STATUS.SUCCESS}
-          setSelectedSignificantTerm={setSelectedSignificantTerm}
-          selectedTerm={{
-            fieldName: selectedHistogram.field,
-            fieldValue: selectedHistogram.value,
-          }}
-          onFilter={onClose}
-        />
+      <div data-test-subj="apmCorrelationsTable">
+        {histograms.length > 0 && selectedHistogram !== undefined && (
+          <CorrelationsTable
+            // @ts-ignore correlations don't have the same column format other tables have
+            columns={mlCorrelationColumns}
+            // @ts-expect-error correlations don't have the same significant term other tables have
+            significantTerms={histogramTerms}
+            status={FETCH_STATUS.SUCCESS}
+            setSelectedSignificantTerm={setSelectedSignificantTerm}
+            selectedTerm={{
+              fieldName: selectedHistogram.field,
+              fieldValue: selectedHistogram.value,
+            }}
+            onFilter={onClose}
+          />
+        )}
+        {histograms.length < 1 && progress > 0.99 ? (
+          <>
+            <EuiSpacer size="m" />
+            <EuiText textAlign="center">
+              <FormattedMessage
+                id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
+                defaultMessage="No significant correlations found"
+              />
+            </EuiText>
+          </>
+        ) : null}
+      </div>
+      {log.length > 0 && displayLog && (
+        <EuiAccordion
+          id="accordion1"
+          buttonContent={i18n.translate(
+            'xpack.apm.correlations.latencyCorrelations.logButtonContent',
+            {
+              defaultMessage: 'Log',
+            }
+          )}
+        >
+          <EuiPanel color="subdued">
+            {log.map((d, i) => {
+              const splitItem = d.split(': ');
+              return (
+                <p key={i}>
+                  <small>
+                    <EuiCode>{splitItem[0]}</EuiCode> {splitItem[1]}
+                  </small>
+                </p>
+              );
+            })}
+          </EuiPanel>
+        </EuiAccordion>
       )}
-      {histograms.length < 1 && progress > 0.99 ? (
-        <>
-          <EuiSpacer size="m" />
-          <EuiText textAlign="center">
-            <FormattedMessage
-              id="xpack.apm.correlations.latencyCorrelations.noCorrelationsText"
-              defaultMessage="No significant correlations found"
-            />
-          </EuiText>
-        </>
-      ) : null}
     </>
   );
 }
diff --git a/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts b/x-pack/plugins/apm/public/components/app/correlations/use_correlations.ts
@@ -21,7 +21,6 @@ import { useKibana } from '../../../../../../../src/plugins/kibana_react/public'
 import { ApmPluginStartDeps } from '../../../plugin';
 
 interface CorrelationsOptions {
-  index: string;
   environment?: string;
   kuery?: string;
   serviceName?: string;
@@ -37,6 +36,7 @@ interface RawResponse {
   values: SearchServiceValue[];
   overallHistogram: HistogramItem[];
   log: string[];
+  ccsWarning: boolean;
 }
 
 export const useCorrelations = (params: CorrelationsOptions) => {
@@ -106,6 +106,8 @@ export const useCorrelations = (params: CorrelationsOptions) => {
   };
 
   return {
+    ccsWarning: rawResponse?.ccsWarning ?? false,
+    log: rawResponse?.log ?? [],
     error,
     histograms: rawResponse?.values ?? [],
     percentileThresholdValue:

diff --git a/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx b/x-pack/plugins/apm/public/components/app/service_inventory/service_list/index.tsx
@@ -117,6 +117,7 @@ export function getServiceColumns({
               )}
               <EuiFlexItem className="apmServiceList__serviceNameContainer">
                 <AppLink
+                  data-test-subj="apmServiceListAppLink"
                   serviceName={serviceName}
                   transactionType={transactionType}
                   className="eui-textTruncate"

diff --git a/x-pack/plugins/apm/public/components/routing/templates/apm_service_template/index.tsx b/x-pack/plugins/apm/public/components/routing/templates/apm_service_template/index.tsx
@@ -93,7 +93,9 @@ function TemplateWithContext({
               <EuiFlexGroup alignItems="center">
                 <EuiFlexItem grow={false}>
                   <EuiTitle size="l">
-                    <>{serviceName}</>
+                    <h1 data-test-subj="apmMainTemplateHeaderServiceName">
+                      {serviceName}
+                    </h1>
                   </EuiTitle>
                 </EuiFlexItem>
                 <EuiFlexItem grow={false}>

diff --git a/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts b/x-pack/plugins/apm/server/lib/search_strategies/correlations/async_search_service.ts
@@ -7,6 +7,7 @@
 
 import { shuffle, range } from 'lodash';
 import type { ElasticsearchClient } from 'src/core/server';
+import type { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices';
 import { fetchTransactionDurationFieldCandidates } from './query_field_candidates';
 import { fetchTransactionDurationFieldValuePairs } from './query_field_value_pairs';
 import { fetchTransactionDurationPercentiles } from './query_percentiles';
@@ -16,6 +17,7 @@ import { fetchTransactionDurationRanges, HistogramItem } from './query_ranges';
 import type {
   AsyncSearchProviderProgress,
   SearchServiceParams,
+  SearchServiceFetchParams,
   SearchServiceValue,
 } from '../../../../common/search_strategies/correlations/types';
 import { computeExpectationsAndRanges } from './utils/aggregation_utils';
@@ -28,11 +30,14 @@ const currentTimeAsString = () => new Date().toISOString();
 
 export const asyncSearchServiceProvider = (
   esClient: ElasticsearchClient,
-  params: SearchServiceParams
+  getApmIndices: () => Promise<ApmIndicesConfig>,
+  searchServiceParams: SearchServiceParams,
+  includeFrozen: boolean
 ) => {
   let isCancelled = false;
   let isRunning = true;
   let error: Error;
+  let ccsWarning = false;
   const log: string[] = [];
   const logMessage = (message: string) =>
     log.push(`${currentTimeAsString()}: ${message}`);
@@ -63,7 +68,15 @@ export const asyncSearchServiceProvider = (
   };
 
   const fetchCorrelations = async () => {
+    let params: SearchServiceFetchParams | undefined;
+
     try {
+      const indices = await getApmIndices();
+      params = {
+        ...searchServiceParams,
+        index: indices['apm_oss.transactionIndices'],
+      };
+
       // 95th percentile to be displayed as a marker in the log log chart
       const {
         totalDocs,
@@ -172,7 +185,7 @@ export const asyncSearchServiceProvider = (
 
       async function* fetchTransactionDurationHistograms() {
         for (const item of shuffle(fieldValuePairs)) {
-          if (item === undefined || isCancelled) {
+          if (params === undefined || item === undefined || isCancelled) {
             isRunning = false;
             return;
           }
@@ -222,10 +235,15 @@ export const asyncSearchServiceProvider = (
               yield undefined;
             }
           } catch (e) {
-            // don't fail the whole process for individual correlation queries, just add the error to the internal log.
+            // don't fail the whole process for individual correlation queries,
+            // just add the error to the internal log and check if we'd want to set the
+            // cross-cluster search compatibility warning to true.
             logMessage(
               `Failed to fetch correlation/kstest for '${item.field}/${item.value}'`
             );
+            if (params?.index.includes(':')) {
+              ccsWarning = true;
+            }
             yield undefined;
           }
         }
@@ -247,6 +265,10 @@ export const asyncSearchServiceProvider = (
       error = e;
     }
 
+    if (error !== undefined && params?.index.includes(':')) {
+      ccsWarning = true;
+    }
+
     isRunning = false;
   };
 
@@ -256,6 +278,7 @@ export const asyncSearchServiceProvider = (
     const sortedValues = values.sort((a, b) => b.correlation - a.correlation);
 
     return {
+      ccsWarning,
       error,
       log,
       isRunning,