[7.x] [Logs UI] Add ML job status callouts to results page (#4… (#48309)

Backports the following commits to 7.x:
 - [Logs UI] Add ML job status callouts to results page (#47642)

x-pack/legacy/plugins/infra/common/log_analysis/log_analysis.ts

@@ -12,10 +12,45 @@ export const jobTypeRT = rt.keyof({
 
 export type JobType = rt.TypeOf<typeof jobTypeRT>;
 
-export const jobStatusRT = rt.keyof({
-  created: null,
-  missing: null,
-  running: null,
-});
+// combines and abstracts job and datafeed status
+export type JobStatus =
+  | 'unknown'
+  | 'missing'
+  | 'initializing'
+  | 'stopped'
+  | 'started'
+  | 'finished'
+  | 'failed';
+
+export type SetupStatus =
+  | 'initializing' // acquiring job statuses to determine setup status
+  | 'unknown' // job status could not be acquired (failed request etc)
+  | 'required' // jobs are missing
+  | 'requiredForReconfiguration' // the configurations don't match the source configurations
+  | 'requiredForUpdate' // the definitions don't match the module definitions
+  | 'pending' // In the process of setting up the module for the first time or retrying, waiting for response
+  | 'succeeded' // setup succeeded, notifying user
+  | 'failed' // setup failed, notifying user
+  | 'hiddenAfterSuccess' // hide the setup screen and we show the results for the first time
+  | 'skipped' // setup hidden because the module is in a correct state already
+  | 'skippedButReconfigurable' // setup hidden even though the job configurations are outdated
+  | 'skippedButUpdatable'; // setup hidden even though the job definitions are outdated
+
+/**
+ * Maps a job status to the possibility that results have already been produced
+ * before this state was reached.
+ */
+export const isJobStatusWithResults = (jobStatus: JobStatus) =>
+  ['started', 'finished', 'stopped', 'failed'].includes(jobStatus);
+
+export const isHealthyJobStatus = (jobStatus: JobStatus) =>
+  ['started', 'finished'].includes(jobStatus);
 
-export type JobStatus = rt.TypeOf<typeof jobStatusRT>;
+/**
+ * Maps a setup status to the possibility that results have already been
+ * produced before this state was reached.
+ */
+export const isSetupStatusWithResults = (setupStatus: SetupStatus) =>
+  ['skipped', 'hiddenAfterSuccess', 'skippedButReconfigurable', 'skippedButUpdatable'].includes(
+    setupStatus
+  );

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export * from './log_analysis_job_problem_indicator';

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/job_configuration_outdated_callout.tsx

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import React from 'react';
+
+import { RecreateJobCallout } from './recreate_job_callout';
+
+export const JobConfigurationOutdatedCallout: React.FC<{
+  onRecreateMlJob: () => void;
+}> = ({ onRecreateMlJob }) => (
+  <RecreateJobCallout title={jobConfigurationOutdatedTitle} onRecreateMlJob={onRecreateMlJob}>
+    <FormattedMessage
+      id="xpack.infra.logs.analysis.jobConfigurationOutdatedCalloutMessage"
+      defaultMessage="The ML job was created using a different source configuration. Recreate the job to apply the current configuration. This removes previously detected anomalies."
+    />
+  </RecreateJobCallout>
+);
+
+const jobConfigurationOutdatedTitle = i18n.translate(
+  'xpack.infra.logs.analysis.jobConfigurationOutdatedCalloutTitle',
+  {
+    defaultMessage: 'ML job configuration outdated',
+  }
+);

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/job_definition_outdated_callout.tsx

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+import React from 'react';
+
+import { RecreateJobCallout } from './recreate_job_callout';
+
+export const JobDefinitionOutdatedCallout: React.FC<{
+  onRecreateMlJob: () => void;
+}> = ({ onRecreateMlJob }) => (
+  <RecreateJobCallout title={jobDefinitionOutdatedTitle} onRecreateMlJob={onRecreateMlJob}>
+    <FormattedMessage
+      id="xpack.infra.logs.analysis.jobDefinitionOutdatedCalloutMessage"
+      defaultMessage="A newer version of the ML job is available. Recreate the job to deploy the newer version. This removes previously detected anomalies."
+    />
+  </RecreateJobCallout>
+);
+
+const jobDefinitionOutdatedTitle = i18n.translate(
+  'xpack.infra.logs.analysis.jobDefinitionOutdatedCalloutTitle',
+  {
+    defaultMessage: 'ML job definition outdated',
+  }
+);

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/job_stopped_callout.tsx

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { EuiCallOut } from '@elastic/eui';
+import { i18n } from '@kbn/i18n';
+import { FormattedMessage } from '@kbn/i18n/react';
+
+export const JobStoppedCallout: React.FC = () => (
+  <EuiCallOut color="primary" iconType="pause" title={jobStoppedTitle}>
+    <FormattedMessage
+      id="xpack.infra.logs.analysis.jobStoppedCalloutMessage"
+      defaultMessage="The ML job has been stopped manually or due to a lack of resources. New log entries will not be processed until the job has been restarted."
+      tagName="p"
+    />
+  </EuiCallOut>
+);
+
+const jobStoppedTitle = i18n.translate('xpack.infra.logs.analysis.jobStoppedCalloutTitle', {
+  defaultMessage: 'ML job stopped',
+});

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/log_analysis_job_problem_indicator.tsx

@@ -0,0 +1,29 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+
+import { JobStatus, SetupStatus } from '../../../../common/log_analysis';
+import { JobConfigurationOutdatedCallout } from './job_configuration_outdated_callout';
+import { JobDefinitionOutdatedCallout } from './job_definition_outdated_callout';
+import { JobStoppedCallout } from './job_stopped_callout';
+
+export const LogAnalysisJobProblemIndicator: React.FC<{
+  jobStatus: JobStatus;
+  setupStatus: SetupStatus;
+  onRecreateMlJobForReconfiguration: () => void;
+  onRecreateMlJobForUpdate: () => void;
+}> = ({ jobStatus, setupStatus, onRecreateMlJobForReconfiguration, onRecreateMlJobForUpdate }) => {
+  if (jobStatus === 'stopped') {
+    return <JobStoppedCallout />;
+  } else if (setupStatus === 'skippedButUpdatable') {
+    return <JobDefinitionOutdatedCallout onRecreateMlJob={onRecreateMlJobForUpdate} />;
+  } else if (setupStatus === 'skippedButReconfigurable') {
+    return <JobConfigurationOutdatedCallout onRecreateMlJob={onRecreateMlJobForReconfiguration} />;
+  }
+
+  return null; // no problem to indicate
+};

x-pack/legacy/plugins/infra/public/components/logging/log_analysis_job_status/recreate_job_callout.tsx

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React from 'react';
+import { EuiCallOut, EuiButton } from '@elastic/eui';
+import { FormattedMessage } from '@kbn/i18n/react';
+
+export const RecreateJobCallout: React.FC<{
+  onRecreateMlJob: () => void;
+  title?: React.ReactNode;
+}> = ({ children, onRecreateMlJob, title }) => (
+  <EuiCallOut color="warning" iconType="alert" title={title}>
+    <p>{children}</p>
+    <EuiButton color="warning" onClick={onRecreateMlJob}>
+      <FormattedMessage
+        id="xpack.infra.logs.analysis.recreateJobButtonLabel"
+        defaultMessage="Recreate ML job"
+      />
+    </EuiButton>
+  </EuiCallOut>
+);

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_api_types.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import * as rt from 'io-ts';
+
+export const jobCustomSettingsRT = rt.partial({
+  job_revision: rt.number,
+  logs_source_config: rt.partial({
+    indexPattern: rt.string,
+    timestampField: rt.string,
+    bucketSpan: rt.number,
+  }),
+});

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_get_jobs_summary_api.ts

@@ -4,11 +4,13 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import * as rt from 'io-ts';
-import { kfetch } from 'ui/kfetch';
 import { pipe } from 'fp-ts/lib/pipeable';
 import { fold } from 'fp-ts/lib/Either';
 import { identity } from 'fp-ts/lib/function';
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
+import { jobCustomSettingsRT } from './ml_api_types';
 import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
 import { getAllModuleJobIds } from '../../../../../common/log_analysis';
 
@@ -56,11 +58,14 @@ export const jobSummaryRT = rt.intersection([
     datafeedIndices: rt.array(rt.string),
     datafeedState: datafeedStateRT,
     fullJob: rt.partial({
+      custom_settings: jobCustomSettingsRT,
       finished_time: rt.number,
     }),
   }),
 ]);
 
+export type JobSummary = rt.TypeOf<typeof jobSummaryRT>;
+
 export const fetchJobStatusResponsePayloadRT = rt.array(jobSummaryRT);
 
 export type FetchJobStatusResponsePayload = rt.TypeOf<typeof fetchJobStatusResponsePayloadRT>;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_get_module.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { fold } from 'fp-ts/lib/Either';
+import { pipe } from 'fp-ts/lib/pipeable';
+import { identity } from 'fp-ts/lib/function';
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
+import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
+import { jobCustomSettingsRT } from './ml_api_types';
+
+export const callGetMlModuleAPI = async (moduleId: string) => {
+  const response = await kfetch({
+    method: 'GET',
+    pathname: `/api/ml/modules/get_module/${moduleId}`,
+  });
+
+  return pipe(
+    getMlModuleResponsePayloadRT.decode(response),
+    fold(throwErrors(createPlainError), identity)
+  );
+};
+
+const jobDefinitionRT = rt.type({
+  id: rt.string,
+  config: rt.type({
+    custom_settings: jobCustomSettingsRT,
+  }),
+});
+
+export type JobDefinition = rt.TypeOf<typeof jobDefinitionRT>;
+
+const getMlModuleResponsePayloadRT = rt.type({
+  id: rt.string,
+  jobs: rt.array(jobDefinitionRT),
+});
+
+export type GetMlModuleResponsePayload = rt.TypeOf<typeof getMlModuleResponsePayloadRT>;

x-pack/legacy/plugins/infra/public/containers/logs/log_analysis/api/ml_setup_module_api.ts

@@ -4,27 +4,18 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import * as rt from 'io-ts';
-import { kfetch } from 'ui/kfetch';
-
 import { fold } from 'fp-ts/lib/Either';
 import { pipe } from 'fp-ts/lib/pipeable';
 import { identity } from 'fp-ts/lib/function';
+import * as rt from 'io-ts';
+import { kfetch } from 'ui/kfetch';
+
 import { throwErrors, createPlainError } from '../../../../../common/runtime_types';
 import { getJobIdPrefix } from '../../../../../common/log_analysis';
-
-const MODULE_ID = 'logs_ui_analysis';
-
-// This is needed due to: https://github.com/elastic/kibana/issues/43671
-const removeSampleDataIndex = (indexPattern: string) => {
-  const SAMPLE_DATA_INDEX = 'kibana_sample_data_logs*';
-  return indexPattern
-    .split(',')
-    .filter(index => index !== SAMPLE_DATA_INDEX)
-    .join(',');
-};
+import { jobCustomSettingsRT } from './ml_api_types';
 
 export const callSetupMlModuleAPI = async (
+  moduleId: string,
   start: number | undefined,
   end: number | undefined,
   spaceId: string,
@@ -35,23 +26,30 @@ export const callSetupMlModuleAPI = async (
 ) => {
   const response = await kfetch({
     method: 'POST',
-    pathname: `/api/ml/modules/setup/${MODULE_ID}`,
+    pathname: `/api/ml/modules/setup/${moduleId}`,
     body: JSON.stringify(
       setupMlModuleRequestPayloadRT.encode({
         start,
         end,
-        indexPatternName: removeSampleDataIndex(indexPattern),
+        indexPatternName: indexPattern,
         prefix: getJobIdPrefix(spaceId, sourceId),
         startDatafeed: true,
         jobOverrides: [
           {
-            job_id: 'log-entry-rate',
+            job_id: 'log-entry-rate' as const,
             analysis_config: {
               bucket_span: `${bucketSpan}ms`,
             },
             data_description: {
               time_field: timeField,
             },
+            custom_settings: {
+              logs_source_config: {
+                indexPattern,
+                timestampField: timeField,
+                bucketSpan,
+              },
+            },
           },
         ],
         datafeedOverrides: [],
@@ -70,11 +68,22 @@ const setupMlModuleTimeParamsRT = rt.partial({
   end: rt.number,
 });
 
+const setupMlModuleLogEntryRateJobOverridesRT = rt.type({
+  job_id: rt.literal('log-entry-rate'),
+  analysis_config: rt.type({
+    bucket_span: rt.string,
+  }),
+  data_description: rt.type({
+    time_field: rt.string,
+  }),
+  custom_settings: jobCustomSettingsRT,
+});
+
 const setupMlModuleRequestParamsRT = rt.type({
   indexPatternName: rt.string,
   prefix: rt.string,
   startDatafeed: rt.boolean,
-  jobOverrides: rt.array(rt.object),
+  jobOverrides: rt.array(setupMlModuleLogEntryRateJobOverridesRT),
   datafeedOverrides: rt.array(rt.object),
 });