Merge branch '7.17' into backport/7.17/pr-182683

elastic · May 14, 2024 · 0801043 · 0801043
2 parents 8f9332e + df05bca
commit 0801043
Show file tree

Hide file tree

Showing 9 changed files with 67 additions and 27 deletions.
diff --git a/.buildkite/pipelines/es_snapshots/build.yml b/.buildkite/pipelines/es_snapshots/build.yml
@@ -4,7 +4,7 @@ steps:
     timeout_in_minutes: 30
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       localSsds: 1
       localSsdInterface: nvme

diff --git a/.buildkite/pipelines/es_snapshots/promote.yml b/.buildkite/pipelines/es_snapshots/promote.yml
@@ -12,6 +12,6 @@ steps:
     command: .buildkite/scripts/steps/es_snapshots/promote.sh
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
diff --git a/.buildkite/pipelines/es_snapshots/verify.yml b/.buildkite/pipelines/es_snapshots/verify.yml
@@ -16,7 +16,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
 
@@ -26,7 +26,7 @@ steps:
     label: Build Kibana Distribution and Plugins
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: c2-standard-16
       preemptible: true
@@ -39,7 +39,7 @@ steps:
     parallelism: 27
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
     depends_on: build
@@ -54,7 +54,7 @@ steps:
     label: 'Docker CI Group'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
     depends_on: build
@@ -70,7 +70,7 @@ steps:
     parallelism: 11
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
       preemptible: true
@@ -87,7 +87,7 @@ steps:
     parallelism: 3
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-4
     timeout_in_minutes: 120
@@ -101,7 +101,7 @@ steps:
     label: 'API Integration Tests'
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
     timeout_in_minutes: 120
@@ -112,7 +112,7 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
     depends_on:
@@ -130,6 +130,6 @@ steps:
     timeout_in_minutes: 10
     agents:
       image: family/kibana-ubuntu-2004
-      imageProject: elastic-images-qa
+      imageProject: elastic-images-prod
       provider: gcp
       machineType: n2-standard-2
diff --git a/.buildkite/pipelines/flaky_tests/pipeline.js b/.buildkite/pipelines/flaky_tests/pipeline.js
@@ -27,7 +27,7 @@ const getAgentRule = (queueName = 'n2-4-spot') => {
     return {
       provider: 'gcp',
       image: 'family/kibana-ubuntu-2004',
-      imageProject: 'elastic-images-qa',
+      imageProject: 'elastic-images-prod',
       machineType: `${kind}-standard-${cores}`,
       ...additionalProps,
     };

diff --git a/.buildkite/pipelines/flaky_tests/runner.js b/.buildkite/pipelines/flaky_tests/runner.js
@@ -27,7 +27,7 @@ const getAgentRule = (queueName = 'n2-4-spot') => {
     return {
       provider: 'gcp',
       image: 'family/kibana-ubuntu-2004',
-      imageProject: 'elastic-images-qa',
+      imageProject: 'elastic-images-prod',
       machineType: `${kind}-standard-${cores}`,
       ...additionalProps,
     };

diff --git a/test/package/Vagrantfile b/test/package/Vagrantfile
@@ -19,6 +19,9 @@ Vagrant.configure("2") do |config|
       vb.cpus = 2
     end
     rpm.vm.box = 'almalinux/9'
+    # `rpm -i` is broken in 9.4 when installing from a shared folder
+    # alternative workaround is to copy /packages/kibana.tar.gz to the cwd
+    rpm.vm.box_version = '9.3.20231118'
     rpm.vm.provision "ansible" do |ansible|
       ansible.playbook = "rpm.yml"
     end

diff --git a/x-pack/plugins/osquery/server/routes/pack/create_pack_route.ts b/x-pack/plugins/osquery/server/routes/pack/create_pack_route.ts
@@ -20,7 +20,7 @@ import { OsqueryAppContext } from '../../lib/osquery_app_context_services';
 import { OSQUERY_INTEGRATION_NAME } from '../../../common';
 import { PLUGIN_ID } from '../../../common';
 import { packSavedObjectType } from '../../../common/types';
-import { convertPackQueriesToSO } from './utils';
+import { convertPackQueriesToSO, getInitialPolicies } from './utils';
 import { getInternalSavedObjectsClient } from '../../usage/collector';
 
 export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
@@ -92,12 +92,22 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte
         }
       )) ?? { items: [] };
 
-      const agentPolicies = policy_ids
-        ? mapKeys(await agentPolicyService?.getByIds(internalSavedObjectsClient, policy_ids), 'id')
+      const { policiesList, invalidPolicies } = getInitialPolicies(packagePolicies, policy_ids);
+
+      if (invalidPolicies?.length) {
+        return response.badRequest({
+          body: `The following policy ids are invalid: ${invalidPolicies.join(', ')}`,
+        });
+      }
+      const agentPolicies = policiesList
+        ? mapKeys(
+            await agentPolicyService?.getByIds(internalSavedObjectsClient, policiesList),
+            'id'
+          )
         : {};
 
-      const references = policy_ids
-        ? policy_ids.map((policyId: string) => ({
+      const references = policiesList
+        ? policiesList.map((policyId: string) => ({
             id: policyId,
             name: agentPolicies[policyId].name,
             type: AGENT_POLICY_SAVED_OBJECT_TYPE,
@@ -122,9 +132,9 @@ export const createPackRoute = (router: IRouter, osqueryContext: OsqueryAppConte
         }
       );
 
-      if (enabled && policy_ids?.length) {
+      if (enabled && policiesList?.length) {
         await Promise.all(
-          policy_ids.map((agentPolicyId) => {
+          policiesList.map((agentPolicyId) => {
             const packagePolicy = find(packagePolicies, ['policy_id', agentPolicyId]);
             if (packagePolicy) {
               return packagePolicyService?.update(

diff --git a/x-pack/plugins/osquery/server/routes/pack/update_pack_route.ts b/x-pack/plugins/osquery/server/routes/pack/update_pack_route.ts
@@ -21,7 +21,7 @@ import { OSQUERY_INTEGRATION_NAME } from '../../../common';
 import { packSavedObjectType } from '../../../common/types';
 import { OsqueryAppContext } from '../../lib/osquery_app_context_services';
 import { PLUGIN_ID } from '../../../common';
-import { convertSOQueriesToPack, convertPackQueriesToSO } from './utils';
+import { convertSOQueriesToPack, convertPackQueriesToSO, getInitialPolicies } from './utils';
 import { getInternalSavedObjectsClient } from '../../usage/collector';
 
 export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppContext) => {
@@ -112,8 +112,20 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte
       const currentPackagePolicies = filter(packagePolicies, (packagePolicy) =>
         has(packagePolicy, `inputs[0].config.osquery.value.packs.${currentPackSO.attributes.name}`)
       );
-      const agentPolicies = policy_ids
-        ? mapKeys(await agentPolicyService?.getByIds(internalSavedObjectsClient, policy_ids), 'id')
+
+      const { policiesList, invalidPolicies } = getInitialPolicies(packagePolicies, policy_ids);
+
+      if (invalidPolicies?.length) {
+        return response.badRequest({
+          body: `The following policy ids are invalid: ${invalidPolicies.join(', ')}`,
+        });
+      }
+
+      const agentPolicies = policiesList
+        ? mapKeys(
+            await agentPolicyService?.getByIds(internalSavedObjectsClient, policiesList),
+            'id'
+          )
         : {};
       const agentPolicyIds = Object.keys(agentPolicies);
 
@@ -128,10 +140,10 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte
           updated_at: moment().toISOString(),
           updated_by: currentUser,
         },
-        policy_ids
+        policiesList
           ? {
               refresh: 'wait_for',
-              references: policy_ids.map((id) => ({
+              references: policiesList.map((id) => ({
                 id,
                 name: agentPolicies[id].name,
                 type: AGENT_POLICY_SAVED_OBJECT_TYPE,
@@ -161,7 +173,7 @@ export const updatePackRoute = (router: IRouter, osqueryContext: OsqueryAppConte
 
       if (enabled != null && enabled !== currentPackSO.attributes.enabled) {
         if (enabled) {
-          const policyIds = policy_ids ? agentPolicyIds : currentAgentPolicyIds;
+          const policyIds = policiesList ? agentPolicyIds : currentAgentPolicyIds;
 
           await Promise.all(
             policyIds.map((agentPolicyId) => {

diff --git a/x-pack/plugins/osquery/server/routes/pack/utils.ts b/x-pack/plugins/osquery/server/routes/pack/utils.ts
@@ -5,7 +5,8 @@
  * 2.0.
  */
 
-import { pick, reduce } from 'lodash';
+import { difference, intersection, map, pick, reduce, uniq } from 'lodash';
+import { PackagePolicy } from '../../../../fleet/common';
 import { convertECSMappingToArray, convertECSMappingToObject } from '../utils';
 
 // @ts-expect-error update types
@@ -40,3 +41,17 @@ export const convertSOQueriesToPack = (queries) =>
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     {} as Record<string, any>
   );
+
+export const getInitialPolicies = (
+  packagePolicies: PackagePolicy[] | never[],
+  policyIds: string[] = []
+): { policiesList: string[]; invalidPolicies?: string[] } => {
+  const supportedPackagePolicyIds = uniq(map(packagePolicies, 'policy_id'));
+  const policiesList = intersection(uniq(policyIds), supportedPackagePolicyIds);
+  const invalidPolicies = difference(uniq(policyIds), policiesList);
+
+  return {
+    policiesList,
+    ...(invalidPolicies.length && { invalidPolicies }),
+  };
+};