[actions] adds proxyBypassHosts and proxyOnlyHosts Kibana config keys

resolves elastic#92949

This PR adds two new Kibana config keys to further customize when the proxy
is used when making HTTP requests.  Prior to this PR, if a proxy was set
via the `xpack.actions.proxyUrl` config key, all requests would be
proxied.

Now, there's a further refinement in that hostnames can be added
to the `xpack.actions.proxyBypassHosts` and `xpack.actions.proxyOnlyHosts`
config keys.  Only one of these config keys can be used at a time.

If the target URL hostname of the HTTP request is listed in the
`proxyBypassHosts` list, the proxy won't be used.

If the target URL hostname of the HTTP request is **NOT** listed in the
`proxyOnlyHosts` list, the proxy won't be used.

Depending on the customer's environment, it may be easier to list the hosts to
bypass, or easier to list the hosts that should only be proxied, so they can
choose either method.

docs/settings/alert-action-settings.asciidoc

@@ -59,6 +59,12 @@ You can configure the following settings in the `kibana.yml` file.
 | `xpack.actions.proxyUrl` {ess-icon}
   | Specifies the proxy URL to use, if using a proxy for actions. By default, no proxy is used.
 
+| `xpack.actions.proxyBypassHosts` {ess-icon}
+  | Specifies hostnames which should not use the proxy, if using a proxy for actions. The value is an array of hostnames as strings.  By default, all hosts will use the proxy.  The settings `xpack.actions.proxyBypassHosts` and `xpack.actions.proxyOnlyHosts` cannot be used at the same time.
+
+| `xpack.actions.proxyOnlyHosts` {ess-icon}
+  | Specifies hostnames which should only use the proxy, if using a proxy for actions. The value is an array of hostnames as strings.  By default, all hosts will use the proxy.  The settings `xpack.actions.proxyBypassHosts` and `xpack.actions.proxyOnlyHosts` cannot be used at the same time.
+
 | `xpack.actions.proxyHeaders` {ess-icon}
   | Specifies HTTP headers for the proxy, if using a proxy for actions. Defaults to {}.
 

src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker

@@ -163,6 +163,8 @@ kibana_vars=(
     xpack.actions.proxyHeaders
     xpack.actions.proxyRejectUnauthorizedCertificates
     xpack.actions.proxyUrl
+    xpack.actions.proxyBypassHosts
+    xpack.actions.proxyOnlyHosts
     xpack.actions.rejectUnauthorized
     xpack.alerts.healthCheck.interval
     xpack.alerts.invalidateApiKeysTask.interval

x-pack/plugins/actions/server/actions_client.test.ts

@@ -406,6 +406,8 @@ describe('create()', () => {
       preconfigured: {},
       proxyRejectUnauthorizedCertificates: true,
       rejectUnauthorized: true,
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: undefined,
     });
 
     const localActionTypeRegistryParams = {

x-pack/plugins/actions/server/actions_config.test.ts

@@ -253,3 +253,82 @@ describe('ensureActionTypeEnabled', () => {
     expect(getActionsConfigurationUtilities(config).ensureActionTypeEnabled('foo')).toBeUndefined();
   });
 });
+
+describe('getProxySettings', () => {
+  test('returns undefined when no proxy URL set', () => {
+    const config: ActionsConfig = {
+      ...defaultActionsConfig,
+      proxyHeaders: { someHeaderName: 'some header value' },
+      proxyBypassHosts: ['avoid-proxy.co'],
+    };
+
+    const proxySettings = getActionsConfigurationUtilities(config).getProxySettings();
+    expect(proxySettings).toBeUndefined();
+  });
+
+  test('returns proxy url', () => {
+    const config: ActionsConfig = {
+      ...defaultActionsConfig,
+      proxyUrl: 'https://proxy.elastic.co',
+    };
+    const proxySettings = getActionsConfigurationUtilities(config).getProxySettings();
+    expect(proxySettings?.proxyUrl).toBe(config.proxyUrl);
+  });
+
+  test('returns proxyRejectUnauthorizedCertificates', () => {
+    const configTrue: ActionsConfig = {
+      ...defaultActionsConfig,
+      proxyUrl: 'https://proxy.elastic.co',
+      proxyRejectUnauthorizedCertificates: true,
+    };
+    let proxySettings = getActionsConfigurationUtilities(configTrue).getProxySettings();
+    expect(proxySettings?.proxyRejectUnauthorizedCertificates).toBe(true);
+
+    const configFalse: ActionsConfig = {
+      ...defaultActionsConfig,
+      proxyUrl: 'https://proxy.elastic.co',
+      proxyRejectUnauthorizedCertificates: false,
+    };
+    proxySettings = getActionsConfigurationUtilities(configFalse).getProxySettings();
+    expect(proxySettings?.proxyRejectUnauthorizedCertificates).toBe(false);
+  });
+
+  test('returns proxy headers', () => {
+    const proxyHeaders = {
+      someHeaderName: 'some header value',
+      someOtherHeader: 'some other header',
+    };
+    const config: ActionsConfig = {
+      ...defaultActionsConfig,
+      proxyUrl: 'https://proxy.elastic.co',
+      proxyHeaders,
+    };
+
+    const proxySettings = getActionsConfigurationUtilities(config).getProxySettings();
+    expect(proxySettings?.proxyHeaders).toEqual(config.proxyHeaders);
+  });
+
+  test('returns proxy bypass hosts', () => {
+    const proxyBypassHosts = ['proxy-bypass-1.elastic.co', 'proxy-bypass-2.elastic.co'];
+    const config: ActionsConfig = {
+      ...defaultActionsConfig,
+      proxyUrl: 'https://proxy.elastic.co',
+      proxyBypassHosts,
+    };
+
+    const proxySettings = getActionsConfigurationUtilities(config).getProxySettings();
+    expect(proxySettings?.proxyBypassHosts).toEqual(new Set(proxyBypassHosts));
+  });
+
+  test('returns proxy only hosts', () => {
+    const proxyOnlyHosts = ['proxy-only-1.elastic.co', 'proxy-only-2.elastic.co'];
+    const config: ActionsConfig = {
+      ...defaultActionsConfig,
+      proxyUrl: 'https://proxy.elastic.co',
+      proxyOnlyHosts,
+    };
+
+    const proxySettings = getActionsConfigurationUtilities(config).getProxySettings();
+    expect(proxySettings?.proxyOnlyHosts).toEqual(new Set(proxyOnlyHosts));
+  });
+});

x-pack/plugins/actions/server/actions_config.ts

@@ -11,17 +11,11 @@ import url from 'url';
 import { curry } from 'lodash';
 import { pipe } from 'fp-ts/lib/pipeable';
 
-import { ActionsConfig } from './config';
+import { ActionsConfig, AllowedHosts, EnabledActionTypes } from './config';
 import { ActionTypeDisabledError } from './lib';
 import { ProxySettings } from './types';
 
-export enum AllowedHosts {
-  Any = '*',
-}
-
-export enum EnabledActionTypes {
-  Any = '*',
-}
+export { AllowedHosts, EnabledActionTypes } from './config';
 
 enum AllowListingField {
   URL = 'url',
@@ -93,11 +87,18 @@ function getProxySettingsFromConfig(config: ActionsConfig): undefined | ProxySet
 
   return {
     proxyUrl: config.proxyUrl,
+    proxyBypassHosts: arrayAsSet(config.proxyBypassHosts),
+    proxyOnlyHosts: arrayAsSet(config.proxyOnlyHosts),
     proxyHeaders: config.proxyHeaders,
     proxyRejectUnauthorizedCertificates: config.proxyRejectUnauthorizedCertificates,
   };
 }
 
+function arrayAsSet<T>(arr: T[] | undefined): Set<T> | undefined {
+  if (!arr) return;
+  return new Set(arr);
+}
+
 export function getActionsConfigurationUtilities(
   config: ActionsConfig
 ): ActionsConfigurationUtilities {

x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.test.ts

@@ -7,12 +7,16 @@
 
 import axios from 'axios';
 import { Agent as HttpsAgent } from 'https';
+import HttpProxyAgent from 'http-proxy-agent';
+import { HttpsProxyAgent } from 'https-proxy-agent';
 import { Logger } from '../../../../../../src/core/server';
 import { addTimeZoneToDate, request, patch, getErrorMessage } from './axios_utils';
 import { loggingSystemMock } from '../../../../../../src/core/server/mocks';
 import { actionsConfigMock } from '../../actions_config.mock';
 import { getCustomAgents } from './get_custom_agents';
 
+const TestUrl = 'https://elastic.co/foo/bar/baz';
+
 const logger = loggingSystemMock.create().get() as jest.Mocked<Logger>;
 const configurationUtilities = actionsConfigMock.create();
 jest.mock('axios');
@@ -66,17 +70,19 @@ describe('request', () => {
     configurationUtilities.getProxySettings.mockReturnValue({
       proxyRejectUnauthorizedCertificates: true,
       proxyUrl: 'https://localhost:1212',
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: undefined,
     });
-    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger);
+    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger, TestUrl);
 
     const res = await request({
       axios,
-      url: 'http://testProxy',
+      url: TestUrl,
       logger,
       configurationUtilities,
     });
 
-    expect(axiosMock).toHaveBeenCalledWith('http://testProxy', {
+    expect(axiosMock).toHaveBeenCalledWith(TestUrl, {
       method: 'get',
       data: {},
       httpAgent,
@@ -94,6 +100,8 @@ describe('request', () => {
     configurationUtilities.getProxySettings.mockReturnValue({
       proxyUrl: ':nope:',
       proxyRejectUnauthorizedCertificates: false,
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: undefined,
     });
     const res = await request({
       axios,
@@ -116,6 +124,90 @@ describe('request', () => {
     });
   });
 
+  test('it bypasses with proxyBypassHosts when expected', async () => {
+    configurationUtilities.getProxySettings.mockReturnValue({
+      proxyRejectUnauthorizedCertificates: true,
+      proxyUrl: 'https://elastic.proxy.co',
+      proxyBypassHosts: new Set(['elastic.co']),
+      proxyOnlyHosts: undefined,
+    });
+
+    await request({
+      axios,
+      url: TestUrl,
+      logger,
+      configurationUtilities,
+    });
+
+    expect(axiosMock.mock.calls.length).toBe(1);
+    const { httpAgent, httpsAgent } = axiosMock.mock.calls[0][1];
+    expect(httpAgent instanceof HttpProxyAgent).toBe(false);
+    expect(httpsAgent instanceof HttpsProxyAgent).toBe(false);
+  });
+
+  test('it does not bypass with proxyBypassHosts when expected', async () => {
+    configurationUtilities.getProxySettings.mockReturnValue({
+      proxyRejectUnauthorizedCertificates: true,
+      proxyUrl: 'https://elastic.proxy.co',
+      proxyBypassHosts: new Set(['not-elastic.co']),
+      proxyOnlyHosts: undefined,
+    });
+
+    await request({
+      axios,
+      url: TestUrl,
+      logger,
+      configurationUtilities,
+    });
+
+    expect(axiosMock.mock.calls.length).toBe(1);
+    const { httpAgent, httpsAgent } = axiosMock.mock.calls[0][1];
+    expect(httpAgent instanceof HttpProxyAgent).toBe(true);
+    expect(httpsAgent instanceof HttpsProxyAgent).toBe(true);
+  });
+
+  test('it proxies with proxyOnlyHosts when expected', async () => {
+    configurationUtilities.getProxySettings.mockReturnValue({
+      proxyRejectUnauthorizedCertificates: true,
+      proxyUrl: 'https://elastic.proxy.co',
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: new Set(['elastic.co']),
+    });
+
+    await request({
+      axios,
+      url: TestUrl,
+      logger,
+      configurationUtilities,
+    });
+
+    expect(axiosMock.mock.calls.length).toBe(1);
+    const { httpAgent, httpsAgent } = axiosMock.mock.calls[0][1];
+    expect(httpAgent instanceof HttpProxyAgent).toBe(true);
+    expect(httpsAgent instanceof HttpsProxyAgent).toBe(true);
+  });
+
+  test('it does not proxy with proxyOnlyHosts when expected', async () => {
+    configurationUtilities.getProxySettings.mockReturnValue({
+      proxyRejectUnauthorizedCertificates: true,
+      proxyUrl: 'https://elastic.proxy.co',
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: new Set(['not-elastic.co']),
+    });
+
+    await request({
+      axios,
+      url: TestUrl,
+      logger,
+      configurationUtilities,
+    });
+
+    expect(axiosMock.mock.calls.length).toBe(1);
+    const { httpAgent, httpsAgent } = axiosMock.mock.calls[0][1];
+    expect(httpAgent instanceof HttpProxyAgent).toBe(false);
+    expect(httpsAgent instanceof HttpsProxyAgent).toBe(false);
+  });
+
   test('it fetch correctly', async () => {
     const res = await request({
       axios,

x-pack/plugins/actions/server/builtin_action_types/lib/axios_utils.ts

@@ -30,7 +30,7 @@ export const request = async <T = unknown>({
   validateStatus?: (status: number) => boolean;
   auth?: AxiosBasicCredentials;
 }): Promise<AxiosResponse> => {
-  const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger);
+  const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger, url);
 
   return await axios(url, {
     ...rest,

x-pack/plugins/actions/server/builtin_action_types/lib/get_custom_agents.test.ts

@@ -14,15 +14,20 @@ import { loggingSystemMock } from '../../../../../../src/core/server/mocks';
 import { actionsConfigMock } from '../../actions_config.mock';
 const logger = loggingSystemMock.create().get() as jest.Mocked<Logger>;
 
+const targetHost = 'elastic.co';
+const targetUrl = `https://${targetHost}/foo/bar/baz`;
+
 describe('getCustomAgents', () => {
   const configurationUtilities = actionsConfigMock.create();
 
   test('get agents for valid proxy URL', () => {
     configurationUtilities.getProxySettings.mockReturnValue({
       proxyUrl: 'https://someproxyhost',
       proxyRejectUnauthorizedCertificates: false,
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: undefined,
     });
-    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger);
+    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger, targetUrl);
     expect(httpAgent instanceof HttpProxyAgent).toBeTruthy();
     expect(httpsAgent instanceof HttpsProxyAgent).toBeTruthy();
   });
@@ -31,15 +36,41 @@ describe('getCustomAgents', () => {
     configurationUtilities.getProxySettings.mockReturnValue({
       proxyUrl: ':nope: not a valid URL',
       proxyRejectUnauthorizedCertificates: false,
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: undefined,
     });
-    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger);
+    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger, targetUrl);
     expect(httpAgent).toBe(undefined);
     expect(httpsAgent instanceof HttpsAgent).toBeTruthy();
   });
 
   test('return default agents for undefined proxy options', () => {
-    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger);
+    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger, targetUrl);
     expect(httpAgent).toBe(undefined);
     expect(httpsAgent instanceof HttpsAgent).toBeTruthy();
   });
+
+  test('returns non-proxy agents for matching proxyBypassHosts', () => {
+    configurationUtilities.getProxySettings.mockReturnValue({
+      proxyUrl: 'https://someproxyhost',
+      proxyRejectUnauthorizedCertificates: false,
+      proxyBypassHosts: new Set([targetHost]),
+      proxyOnlyHosts: undefined,
+    });
+    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger, targetUrl);
+    expect(httpAgent instanceof HttpProxyAgent).toBeFalsy();
+    expect(httpsAgent instanceof HttpsProxyAgent).toBeFalsy();
+  });
+
+  test('returns proxy agents for matching proxyOnlyHosts', () => {
+    configurationUtilities.getProxySettings.mockReturnValue({
+      proxyUrl: 'https://someproxyhost',
+      proxyRejectUnauthorizedCertificates: false,
+      proxyBypassHosts: undefined,
+      proxyOnlyHosts: new Set([targetHost]),
+    });
+    const { httpAgent, httpsAgent } = getCustomAgents(configurationUtilities, logger, targetUrl);
+    expect(httpAgent instanceof HttpProxyAgent).toBeTruthy();
+    expect(httpsAgent instanceof HttpsProxyAgent).toBeTruthy();
+  });
 });