[Logs UI] Provide index name pattern choice during ML job setup (#48231)

This fixes #48219 by adding the option for the user to select a subset of the configured log indices during the setup process. It also surfaces the errors returned by Kibana when the setup fails.

x-pack/legacy/plugins/infra/common/log_analysis/log_analysis.ts

@@ -54,3 +54,8 @@ export const isSetupStatusWithResults = (setupStatus: SetupStatus) =>
   ['skipped', 'hiddenAfterSuccess', 'skippedButReconfigurable', 'skippedButUpdatable'].includes(
     setupStatus
   );
+
+const KIBANA_SAMPLE_DATA_INDICES = ['kibana_sample_data_logs*'];
+
+export const isExampleDataIndex = (indexName: string) =>
+  KIBANA_SAMPLE_DATA_INDICES.includes(indexName);

x-pack/legacy/plugins/infra/common/utility_types.ts

@@ -11,6 +11,11 @@ export type Pick3<T, K1 extends keyof T, K2 extends keyof T[K1], K3 extends keyo
   [P1 in K1]: { [P2 in K2]: { [P3 in K3]: ((T[K1])[K2])[P3] } };
 };
 
+export type MandatoryProperty<T, Prop extends keyof T> = T &
+  {
+    [prop in Prop]-?: NonNullable<T[Prop]>;
+  };
+
 /**
  * Portions of below code are derived from https://github.com/tycho01/typical
  * under the MIT License

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts

@@ -92,22 +92,34 @@ const setupMlModuleRequestPayloadRT = rt.intersection([
   setupMlModuleRequestParamsRT,
 ]);
 
+const setupErrorResponseRT = rt.type({
+  msg: rt.string,
+});
+
+const datafeedSetupResponseRT = rt.intersection([
+  rt.type({
+    id: rt.string,
+    started: rt.boolean,
+    success: rt.boolean,
+  }),
+  rt.partial({
+    error: setupErrorResponseRT,
+  }),
+]);
+
+const jobSetupResponseRT = rt.intersection([
+  rt.type({
+    id: rt.string,
+    success: rt.boolean,
+  }),
+  rt.partial({
+    error: setupErrorResponseRT,
+  }),
+]);
+
 const setupMlModuleResponsePayloadRT = rt.type({
-  datafeeds: rt.array(
-    rt.type({
-      id: rt.string,
-      started: rt.boolean,
-      success: rt.boolean,
-      error: rt.any,
-    })
-  ),
-  jobs: rt.array(
-    rt.type({
-      id: rt.string,
-      success: rt.boolean,
-      error: rt.any,
-    })
-  ),
+  datafeeds: rt.array(datafeedSetupResponseRT),
+  jobs: rt.array(jobSetupResponseRT),
 });
 
 export type SetupMlModuleResponsePayload = rt.TypeOf<typeof setupMlModuleResponsePayloadRT>;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_jobs.tsx

@@ -16,7 +16,6 @@ import { useLogAnalysisCleanup } from './log_analysis_cleanup';
 import { useStatusState } from './log_analysis_status_state';
 
 const MODULE_ID = 'logs_ui_analysis';
-const SAMPLE_DATA_INDEX = 'kibana_sample_data_logs*';
 
 export const useLogAnalysisJobs = ({
   indexPattern,
@@ -29,11 +28,10 @@ export const useLogAnalysisJobs = ({
   spaceId: string;
   timeField: string;
 }) => {
-  const filteredIndexPattern = useMemo(() => removeSampleDataIndex(indexPattern), [indexPattern]);
   const { cleanupMLResources } = useLogAnalysisCleanup({ sourceId, spaceId });
   const [statusState, dispatch] = useStatusState({
     bucketSpan,
-    indexPattern: filteredIndexPattern,
+    indexPattern,
     timestampField: timeField,
   });
 
@@ -62,15 +60,19 @@ export const useLogAnalysisJobs = ({
   const [setupMlModuleRequest, setupMlModule] = useTrackedPromise(
     {
       cancelPreviousOn: 'resolution',
-      createPromise: async (start, end) => {
+      createPromise: async (
+        indices: string[],
+        start: number | undefined,
+        end: number | undefined
+      ) => {
         dispatch({ type: 'startedSetup' });
         return await callSetupMlModuleAPI(
           MODULE_ID,
           start,
           end,
           spaceId,
           sourceId,
-          filteredIndexPattern,
+          indices.join(','),
           timeField,
           bucketSpan
         );
@@ -82,7 +84,7 @@ export const useLogAnalysisJobs = ({
         dispatch({ type: 'failedSetup' });
       },
     },
-    [filteredIndexPattern, spaceId, sourceId, timeField, bucketSpan]
+    [spaceId, sourceId, timeField, bucketSpan]
   );
 
   const [fetchJobStatusRequest, fetchJobStatus] = useTrackedPromise(
@@ -99,7 +101,7 @@ export const useLogAnalysisJobs = ({
         dispatch({ type: 'failedFetchingJobStatuses' });
       },
     },
-    [filteredIndexPattern, spaceId, sourceId]
+    [spaceId, sourceId]
   );
 
   const isLoadingSetupStatus = useMemo(
@@ -108,16 +110,18 @@ export const useLogAnalysisJobs = ({
     [fetchJobStatusRequest.state, fetchModuleDefinitionRequest.state]
   );
 
+  const availableIndices = useMemo(() => indexPattern.split(','), [indexPattern]);
+
   const viewResults = useCallback(() => {
     dispatch({ type: 'viewedResults' });
   }, []);
 
   const cleanupAndSetup = useCallback(
-    (start, end) => {
+    (indices: string[], start: number | undefined, end: number | undefined) => {
       dispatch({ type: 'startedSetup' });
       cleanupMLResources()
         .then(() => {
-          setupMlModule(start, end);
+          setupMlModule(indices, start, end);
         })
         .catch(() => {
           dispatch({ type: 'failedSetup' });
@@ -145,9 +149,11 @@ export const useLogAnalysisJobs = ({
   }, [sourceId, spaceId]);
 
   return {
+    availableIndices,
     fetchJobStatus,
     isLoadingSetupStatus,
     jobStatus: statusState.jobStatus,
+    lastSetupErrorMessages: statusState.lastSetupErrorMessages,
     cleanupAndSetup,
     setup: setupMlModule,
     setupMlModuleRequest,
@@ -160,11 +166,3 @@ export const useLogAnalysisJobs = ({
 };
 
 export const LogAnalysisJobs = createContainer(useLogAnalysisJobs);
-//
-// This is needed due to: https://github.com/elastic/kibana/issues/43671
-const removeSampleDataIndex = (indexPattern: string) => {
-  return indexPattern
-    .split(',')
-    .filter(index => index !== SAMPLE_DATA_INDEX)
-    .join(',');
-};

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_setup_state.tsx

@@ -3,35 +3,81 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { useState, useCallback } from 'react';
 
-type SetupHandler = (startTime?: number | undefined, endTime?: number | undefined) => void;
+import { useState, useCallback, useMemo } from 'react';
 
-interface Props {
+import { isExampleDataIndex } from '../../../../common/log_analysis';
+
+type SetupHandler = (
+  indices: string[],
+  startTime: number | undefined,
+  endTime: number | undefined
+) => void;
+
+interface AnalysisSetupStateArguments {
+  availableIndices: string[];
   cleanupAndSetupModule: SetupHandler;
   setupModule: SetupHandler;
 }
 
+type IndicesSelection = Record<string, boolean>;
+
+type ValidationErrors = 'TOO_FEW_SELECTED_INDICES';
+
 const fourWeeksInMs = 86400000 * 7 * 4;
 
-export const useAnalysisSetupState = ({ setupModule, cleanupAndSetupModule }: Props) => {
+export const useAnalysisSetupState = ({
+  availableIndices,
+  cleanupAndSetupModule,
+  setupModule,
+}: AnalysisSetupStateArguments) => {
   const [startTime, setStartTime] = useState<number | undefined>(Date.now() - fourWeeksInMs);
   const [endTime, setEndTime] = useState<number | undefined>(undefined);
 
+  const [selectedIndices, setSelectedIndices] = useState<IndicesSelection>(
+    availableIndices.reduce(
+      (indexMap, indexName) => ({
+        ...indexMap,
+        [indexName]: !(availableIndices.length > 1 && isExampleDataIndex(indexName)),
+      }),
+      {}
+    )
+  );
+
+  const selectedIndexNames = useMemo(
+    () =>
+      Object.entries(selectedIndices)
+        .filter(([_indexName, isSelected]) => isSelected)
+        .map(([indexName]) => indexName),
+    [selectedIndices]
+  );
+
   const setup = useCallback(() => {
-    return setupModule(startTime, endTime);
-  }, [setupModule, startTime, endTime]);
+    return setupModule(selectedIndexNames, startTime, endTime);
+  }, [setupModule, selectedIndexNames, startTime, endTime]);
 
   const cleanupAndSetup = useCallback(() => {
-    return cleanupAndSetupModule(startTime, endTime);
-  }, [cleanupAndSetupModule, startTime, endTime]);
+    return cleanupAndSetupModule(selectedIndexNames, startTime, endTime);
+  }, [cleanupAndSetupModule, selectedIndexNames, startTime, endTime]);
+
+  const validationErrors: ValidationErrors[] = useMemo(
+    () =>
+      Object.values(selectedIndices).some(isSelected => isSelected)
+        ? []
+        : ['TOO_FEW_SELECTED_INDICES' as const],
+    [selectedIndices]
+  );
 
   return {
     cleanupAndSetup,
     endTime,
+    selectedIndexNames,
+    selectedIndices,
     setEndTime,
+    setSelectedIndices,
     setStartTime,
     setup,
     startTime,
+    validationErrors,
   };
 };

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/log_analysis_status_state.tsx

@@ -18,11 +18,13 @@ import {
 import { FetchJobStatusResponsePayload, JobSummary } from './api/ml_get_jobs_summary_api';
 import { GetMlModuleResponsePayload, JobDefinition } from './api/ml_get_module';
 import { SetupMlModuleResponsePayload } from './api/ml_setup_module_api';
+import { MandatoryProperty } from '../../../../common/utility_types';
 
 interface StatusReducerState {
   jobDefinitions: JobDefinition[];
   jobStatus: Record<JobType, JobStatus>;
   jobSummaries: JobSummary[];
+  lastSetupErrorMessages: string[];
   setupStatus: SetupStatus;
   sourceConfiguration: JobSourceConfiguration;
 }
@@ -69,6 +71,7 @@ const createInitialState = (sourceConfiguration: JobSourceConfiguration): Status
     'log-entry-rate': 'unknown',
   },
   jobSummaries: [],
+  lastSetupErrorMessages: [],
   setupStatus: 'initializing',
   sourceConfiguration,
 });
@@ -101,9 +104,18 @@ function statusReducer(state: StatusReducerState, action: StatusReducerAction):
       )
         ? 'succeeded'
         : 'failed';
+      const nextErrorMessages = [
+        ...Object.values(datafeeds)
+          .filter(hasError)
+          .map(datafeed => datafeed.error.msg),
+        ...Object.values(jobs)
+          .filter(hasError)
+          .map(job => job.error.msg),
+      ];
       return {
         ...state,
         jobStatus: nextJobStatus,
+        lastSetupErrorMessages: nextErrorMessages,
         setupStatus: nextSetupStatus,
       };
     }
@@ -348,11 +360,22 @@ const isJobConfigurationConsistent = (
       return (
         jobConfiguration &&
         jobConfiguration.bucketSpan === sourceConfiguration.bucketSpan &&
-        jobConfiguration.indexPattern === sourceConfiguration.indexPattern &&
+        jobConfiguration.indexPattern &&
+        isIndexPatternSubset(jobConfiguration.indexPattern, sourceConfiguration.indexPattern) &&
         jobConfiguration.timestampField === sourceConfiguration.timestampField
       );
     });
 
+const isIndexPatternSubset = (indexPatternSubset: string, indexPatternSuperset: string) => {
+  const subsetSubPatterns = indexPatternSubset.split(',');
+  const supersetSubPatterns = new Set(indexPatternSuperset.split(','));
+
+  return subsetSubPatterns.every(subPattern => supersetSubPatterns.has(subPattern));
+};
+
+const hasError = <Value extends any>(value: Value): value is MandatoryProperty<Value, 'error'> =>
+  value.error != null;
+
 export const useStatusState = (sourceConfiguration: JobSourceConfiguration) => {
   return useReducer(statusReducer, sourceConfiguration, createInitialState);
 };

x-pack/legacy/plugins/infra/public/pages/logs/analysis/page_content.tsx

@@ -17,12 +17,18 @@ import { AnalysisUnavailableContent } from './page_unavailable_content';
 import { AnalysisSetupStatusUnknownContent } from './page_setup_status_unknown';
 
 export const AnalysisPageContent = () => {
-  const { sourceId, source } = useContext(Source.Context);
+  const { sourceId } = useContext(Source.Context);
   const { hasLogAnalysisCapabilites } = useContext(LogAnalysisCapabilities.Context);
 
-  const { setup, cleanupAndSetup, setupStatus, viewResults, fetchJobStatus } = useContext(
-    LogAnalysisJobs.Context
-  );
+  const {
+    availableIndices,
+    cleanupAndSetup,
+    fetchJobStatus,
+    lastSetupErrorMessages,
+    setup,
+    setupStatus,
+    viewResults,
+  } = useContext(LogAnalysisJobs.Context);
 
   useEffect(() => {
     fetchJobStatus();
@@ -50,10 +56,11 @@ export const AnalysisPageContent = () => {
   } else {
     return (
       <AnalysisSetupContent
-        setup={setup}
+        availableIndices={availableIndices}
         cleanupAndSetup={cleanupAndSetup}
+        errorMessages={lastSetupErrorMessages}
+        setup={setup}
         setupStatus={setupStatus}
-        indexPattern={source ? source.configuration.logAlias : ''}
         viewResults={viewResults}
       />
     );