[Fleet] Create Synthetics migration for 8.8.0 (#154952)

## Summary Resolves #155215 Resolves #142653 Handles two primary migrations for Synthetics integration policies. 1. Rounds deprecated schedules to supported schedules 2. Transforms old deprecated throttling schema `5u/3u/20l` to support JSON schema `{ download: 5, upload: 3, latency: 20 }` Schedule migration --- Before 16m schedule <img width="200" alt="Screen Shot 2023-04-19 at 6 17 35 PM" src="https://user-images.githubusercontent.com/11356435/233211993-086ce51e-a056-407f-900b-15638238a243.png"> After <img width="214" alt="Screen Shot 2023-04-19 at 6 44 08 PM" src="https://user-images.githubusercontent.com/11356435/233215688-e049e2b7-6958-4376-9362-a192aa8e94db.png"> Before 4m schedule <img width="190" alt="Screen Shot 2023-04-19 at 6 17 29 PM" src="https://user-images.githubusercontent.com/11356435/233211995-6b7b2927-6851-4947-ab8c-7fb041da03e1.png"> After <img width="208" alt="Screen Shot 2023-04-19 at 6 44 30 PM" src="https://user-images.githubusercontent.com/11356435/233215685-89f093e6-3308-4103-9b98-5acb7a9b5a0a.png"> Before 8m schedule <img width="202" alt="Screen Shot 2023-04-19 at 6 17 23 PM" src="https://user-images.githubusercontent.com/11356435/233211997-9a60c54c-4867-4ccd-a8eb-5b62d59060bf.png"> After <img width="201" alt="Screen Shot 2023-04-19 at 6 44 22 PM" src="https://user-images.githubusercontent.com/11356435/233215687-f3a900cf-b3ec-44f5-b84b-d292dbad623d.png"> Before 2m schedule <img width="193" alt="Screen Shot 2023-04-19 at 6 17 16 PM" src="https://user-images.githubusercontent.com/11356435/233211999-3d42ad71-b72b-4876-911e-5d79564f2351.png"> After <img width="194" alt="Screen Shot 2023-04-19 at 6 43 55 PM" src="https://user-images.githubusercontent.com/11356435/233215690-fd3f13dc-5e32-4904-b804-cbcfcabf0760.png"> Throttling migration --- Before throttling: false <img width="163" alt="Screen Shot 2023-04-19 at 6 17 00 PM" src="https://user-images.githubusercontent.com/11356435/233212002-3a891b25-fc2e-4cce-a730-abf8695d5423.png"> After <img width="185" alt="Screen Shot 2023-04-19 at 6 49 50 PM" src="https://user-images.githubusercontent.com/11356435/233216370-eed97645-26e8-44f2-8f72-8d1e19f39c35.png"> Before custom throttling <img width="274" alt="Screen Shot 2023-04-19 at 6 16 54 PM" src="https://user-images.githubusercontent.com/11356435/233212004-a9fe82fc-d23a-4d54-a5ad-20971c3df211.png"> After <img width="169" alt="Screen Shot 2023-04-19 at 6 49 44 PM" src="https://user-images.githubusercontent.com/11356435/233216389-f2803ca9-3429-4f85-b1ee-0f5a8e02db92.png"> Before default throttling <img width="212" alt="Screen Shot 2023-04-19 at 6 16 48 PM" src="https://user-images.githubusercontent.com/11356435/233212007-d9fbcae7-6f6e-4cd0-b909-629cfc72b7dc.png"> After <img width="181" alt="Screen Shot 2023-04-19 at 6 49 35 PM" src="https://user-images.githubusercontent.com/11356435/233216412-70336d0e-1e45-4809-bd92-c4524a144b99.png"> ### Testing 1. Check out the 8.7.0 branch 2. Create a Synthetics private location at `app/synthetics/settings/private-locations`. 3. Create a monitor, configured at that private location, with invalid schedules, ideally 1 or each type (http, icmp, browser, tcp) with an invalid schedule (for example, 2, 8, 11, 16, 333, etc) at `app/uptime/add-monitor`. Note: you must use Uptime to create monitors with arbitrary schedules. The Synthetics app will not let you. 4. Create a browser monitor, configured with your private location, with throttling turned off. 5. Create a browser monitor, configured for your private location, with a custom throttling profile. 6. Check out this PR and wait for saved object migration to run 7. Navigate to the agent policy for your monitor. Confirm the schedules were updated to supported schedules. Confirm the throttling configs now appear in yaml. Confirm that `throttling: false` remains for the throttling config that was turned off.
elastic · Apr 21, 2023 · b494716 · b494716
1 parent 3c21b45
commit b494716
Show file tree

Hide file tree

Showing 17 changed files with 1,538 additions and 27 deletions.
diff --git a/x-pack/plugins/fleet/server/saved_objects/index.ts b/x-pack/plugins/fleet/server/saved_objects/index.ts
@@ -54,10 +54,8 @@ import {
   migrateInstallationToV860,
   migratePackagePolicyToV860,
 } from './migrations/to_v8_6_0';
-import {
-  migratePackagePolicyToV870,
-  migratePackagePolicyToV880,
-} from './migrations/security_solution';
+import { migratePackagePolicyToV870 } from './migrations/security_solution';
+import { migratePackagePolicyToV880 } from './migrations/to_v8_8_0';
 
 /*
  * Saved object types and mappings

diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/fixtures/8.7.0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/fixtures/8.7.0.ts
diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/index.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/index.ts
@@ -0,0 +1,8 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export { migratePackagePolicyToV880 } from './to_v8_8_0';
diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/to_v8_8_0.test.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/to_v8_8_0.test.ts
@@ -0,0 +1,167 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectMigrationContext } from '@kbn/core/server';
+
+import { getBrowserPolicy, httpPolicy, icmpPolicy, tcpPolicy } from './fixtures/8.7.0';
+
+import { migratePackagePolicyToV880 as migration } from './to_v8_8_0';
+
+describe('8.8.0 Synthetics Package Policy migration', () => {
+  describe('schedule migration', () => {
+    const testSchedules = [
+      ['4', '3'],
+      ['4.5', '5'],
+      ['7', '5'],
+      ['8', '10'],
+      ['9.5', '10'],
+      ['12', '10'],
+      ['13', '15'],
+      ['16', '15'],
+      ['18', '20'],
+      ['21', '20'],
+      ['25', '20'],
+      ['26', '30'],
+      ['31', '30'],
+      ['45', '30'],
+      ['46', '60'],
+      ['61', '60'],
+      ['90', '60'],
+      ['91', '120'],
+      ['121', '120'],
+      ['195', '240'],
+      ['600', '240'],
+    ];
+
+    it.each(testSchedules)('handles a variety of schedules', (invalidSchedule, validSchedule) => {
+      const actual = migration(
+        {
+          ...httpPolicy,
+          attributes: {
+            ...httpPolicy.attributes,
+            inputs: [
+              {
+                ...httpPolicy.attributes.inputs[0],
+                streams: [
+                  {
+                    ...httpPolicy.attributes.inputs[0].streams[0],
+                    vars: {
+                      ...httpPolicy.attributes.inputs[0].streams[0].vars,
+                      schedule: {
+                        value: `"@every ${invalidSchedule}m"`,
+                        type: 'text',
+                      },
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        },
+        {} as SavedObjectMigrationContext
+      );
+      expect(actual.attributes?.inputs[0]?.streams[0]?.vars?.schedule?.value).toEqual(
+        `"@every ${validSchedule}m"`
+      );
+      expect(actual.attributes?.inputs[0]?.streams[0]?.compiled_stream?.schedule).toEqual(
+        `@every ${validSchedule}m`
+      );
+    });
+
+    it('handles browserPolicy with 2 minute', () => {
+      const actual = migration(getBrowserPolicy(), {} as SavedObjectMigrationContext);
+      expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.schedule?.value).toEqual(
+        '"@every 1m"'
+      );
+      expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream?.schedule).toEqual(
+        `@every 1m`
+      );
+    });
+
+    it('handles httpPolicy with 4 minute schedule', () => {
+      const actual = migration(httpPolicy, {} as SavedObjectMigrationContext);
+      expect(actual.attributes?.inputs[0]?.streams[0]?.vars?.schedule?.value).toEqual(
+        '"@every 3m"'
+      );
+      expect(actual.attributes?.inputs[0]?.streams[0]?.compiled_stream?.schedule).toEqual(
+        `@every 3m`
+      );
+    });
+
+    it('handles tcp with 8 minute schedule', () => {
+      const actual = migration(tcpPolicy, {} as SavedObjectMigrationContext);
+      expect(actual.attributes?.inputs[1]?.streams[0]?.vars?.schedule?.value).toEqual(
+        '"@every 10m"'
+      );
+      expect(actual.attributes?.inputs[1]?.streams[0]?.compiled_stream?.schedule).toEqual(
+        `@every 10m`
+      );
+    });
+
+    it('handles icmpPolicy with 16 minute schedule', () => {
+      const actual = migration(icmpPolicy, {} as SavedObjectMigrationContext);
+      expect(actual.attributes?.inputs[2]?.streams[0]?.vars?.schedule?.value).toEqual(
+        '"@every 15m"'
+      );
+      expect(actual.attributes?.inputs[2]?.streams[0]?.compiled_stream?.schedule).toEqual(
+        `@every 15m`
+      );
+    });
+  });
+
+  describe('throttling migration', () => {
+    it('handles throtling config for throttling: false', () => {
+      const actual = migration(getBrowserPolicy('false'), {} as SavedObjectMigrationContext);
+      expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
+        'false'
+      );
+      expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream?.throttling).toEqual(false);
+    });
+
+    it('handles throttling config for default  throttling', () => {
+      const actual = migration(getBrowserPolicy(), {} as SavedObjectMigrationContext);
+      expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
+        JSON.stringify({ download: 5, upload: 3, latency: 20 })
+      );
+      expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream.throttling).toEqual({
+        download: 5,
+        upload: 3,
+        latency: 20,
+      });
+    });
+
+    it('handles throttling config for custom throttling', () => {
+      const actual = migration(
+        getBrowserPolicy('1.6d/0.75u/150l'),
+        {} as SavedObjectMigrationContext
+      );
+      expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
+        JSON.stringify({ download: 1.6, upload: 0.75, latency: 150 })
+      );
+      expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream.throttling).toEqual({
+        download: 1.6,
+        upload: 0.75,
+        latency: 150,
+      });
+    });
+
+    it('handles edge cases', () => {
+      const actual = migration(
+        getBrowserPolicy('not a valid value'),
+        {} as SavedObjectMigrationContext
+      );
+      expect(actual.attributes?.inputs[3]?.streams[0]?.vars?.['throttling.config']?.value).toEqual(
+        JSON.stringify({ download: 5, upload: 3, latency: 20 })
+      );
+      expect(actual.attributes?.inputs[3]?.streams[0]?.compiled_stream.throttling).toEqual({
+        download: 5,
+        upload: 3,
+        latency: 20,
+      });
+    });
+  });
+});
diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/to_v8_8_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/synthetics/to_v8_8_0.ts
@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectMigrationFn, SavedObjectUnsanitizedDoc } from '@kbn/core/server';
+
+import type { PackagePolicy } from '../../../../common';
+
+export const ALLOWED_SCHEDULES_IN_MINUTES = [
+  '1',
+  '3',
+  '5',
+  '10',
+  '15',
+  '20',
+  '30',
+  '60',
+  '120',
+  '240',
+];
+
+export const migratePackagePolicyToV880: SavedObjectMigrationFn<PackagePolicy, PackagePolicy> = (
+  packagePolicyDoc
+) => {
+  if (packagePolicyDoc.attributes.package?.name !== 'synthetics') {
+    return packagePolicyDoc;
+  }
+
+  const updatedPackagePolicyDoc: SavedObjectUnsanitizedDoc<PackagePolicy> = packagePolicyDoc;
+
+  const enabledInput = updatedPackagePolicyDoc.attributes.inputs.find(
+    (input) => input.enabled === true
+  );
+  const enabledStream = enabledInput?.streams.find((stream) => {
+    return ['browser', 'http', 'icmp', 'tcp'].includes(stream.data_stream.dataset);
+  });
+  if (!enabledStream) {
+    return updatedPackagePolicyDoc;
+  }
+
+  if (
+    enabledStream.vars &&
+    enabledStream.vars.schedule?.value &&
+    enabledStream.compiled_stream?.schedule
+  ) {
+    const schedule = enabledStream.vars.schedule.value.match(/\d+\.?\d*/g)?.[0];
+    const updatedSchedule = getNearestSupportedSchedule(schedule);
+    const formattedUpdatedSchedule = `@every ${updatedSchedule}m`;
+    enabledStream.vars.schedule.value = `"${formattedUpdatedSchedule}"`;
+    enabledStream.compiled_stream.schedule = formattedUpdatedSchedule;
+  }
+
+  if (
+    enabledStream.data_stream.dataset === 'browser' &&
+    enabledStream.vars?.['throttling.config'] &&
+    enabledStream.compiled_stream?.throttling
+  ) {
+    const throttling = enabledStream.vars['throttling.config'].value;
+    if (throttling) {
+      const formattedThrottling = handleThrottling(throttling);
+      enabledStream.vars['throttling.config'].value = JSON.stringify(formattedThrottling);
+      enabledStream.compiled_stream.throttling = formattedThrottling;
+    }
+  }
+
+  return updatedPackagePolicyDoc;
+};
+
+const handleThrottling = (
+  throttling: string
+): { download: number; upload: number; latency: number } => {
+  try {
+    const [download = 5, upload = 3, latency = 20] = throttling.match(/\d+\.?\d*/g) || [];
+    return {
+      download: Number(download),
+      upload: Number(upload),
+      latency: Number(latency),
+    };
+  } catch {
+    return {
+      download: 5,
+      upload: 3,
+      latency: 20,
+    };
+  }
+};
+
+const getNearestSupportedSchedule = (currentSchedule: string): string => {
+  try {
+    const closest = ALLOWED_SCHEDULES_IN_MINUTES.reduce(function (prev, curr) {
+      const supportedSchedule = parseFloat(curr);
+      const currSchedule = parseFloat(currentSchedule);
+      const prevSupportedSchedule = parseFloat(prev);
+      return Math.abs(supportedSchedule - currSchedule) <
+        Math.abs(prevSupportedSchedule - currSchedule)
+        ? curr
+        : prev;
+    });
+
+    return closest;
+  } catch {
+    return '10';
+  }
+};
diff --git a/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_8_0.ts b/x-pack/plugins/fleet/server/saved_objects/migrations/to_v8_8_0.ts
@@ -0,0 +1,35 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectMigrationFn } from '@kbn/core/server';
+
+import type { PackagePolicy } from '../../../common';
+
+import { migratePackagePolicyToV880 as SecSolMigratePackagePolicyToV880 } from './security_solution';
+import { migratePackagePolicyToV880 as SyntheticsMigratePackagePolicyToV880 } from './synthetics';
+
+export const migratePackagePolicyToV880: SavedObjectMigrationFn<PackagePolicy, PackagePolicy> = (
+  packagePolicyDoc,
+  migrationContext
+) => {
+  let updatedPackagePolicyDoc = packagePolicyDoc;
+
+  // Endpoint specific migrations
+  if (packagePolicyDoc.attributes.package?.name === 'endpoint') {
+    updatedPackagePolicyDoc = SecSolMigratePackagePolicyToV880(packagePolicyDoc, migrationContext);
+  }
+
+  // Synthetics specific migrations
+  if (packagePolicyDoc.attributes.package?.name === 'synthetics') {
+    updatedPackagePolicyDoc = SyntheticsMigratePackagePolicyToV880(
+      packagePolicyDoc,
+      migrationContext
+    );
+  }
+
+  return updatedPackagePolicyDoc;
+};
diff --git a/x-pack/plugins/fleet/server/types/models/package_policy.ts b/x-pack/plugins/fleet/server/types/models/package_policy.ts
@@ -99,6 +99,7 @@ const PackagePolicyBaseSchema = {
   namespace: NamespaceSchema,
   policy_id: schema.string(),
   enabled: schema.boolean(),
+  is_managed: schema.maybe(schema.boolean()),
   package: schema.maybe(
     schema.object({
       name: schema.string(),

diff --git a/x-pack/plugins/synthetics/common/constants/monitor_defaults.ts b/x-pack/plugins/synthetics/common/constants/monitor_defaults.ts
@@ -48,9 +48,11 @@ export const CUSTOM_LABEL = i18n.translate('xpack.synthetics.connectionProfile.c
   defaultMessage: 'Custom',
 });
 
+export const DEFAULT_THROTTLING_VALUE = { download: '5', upload: '3', latency: '20' };
+
 export const PROFILE_VALUES: ThrottlingConfig[] = [
   {
-    value: { download: '5', upload: '3', latency: '20' },
+    value: DEFAULT_THROTTLING_VALUE,
     id: PROFILE_VALUES_ENUM.DEFAULT,
     label: i18n.translate('xpack.synthetics.connectionProfile.default', {
       defaultMessage: 'Default',