diff --git a/x-pack/plugins/ingest_manager/common/constants/output.ts b/x-pack/plugins/ingest_manager/common/constants/output.ts index 6060a2b63fc8ea..4c22d0e3fe7a30 100644 --- a/x-pack/plugins/ingest_manager/common/constants/output.ts +++ b/x-pack/plugins/ingest_manager/common/constants/output.ts @@ -12,5 +12,4 @@ export const DEFAULT_OUTPUT = { is_default: true, type: OutputType.Elasticsearch, hosts: [''], - api_key: '', }; diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent.ts b/x-pack/plugins/ingest_manager/common/types/models/agent.ts index 14b2b2e47d17fd..fcd3955f3a32fc 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent.ts @@ -20,15 +20,18 @@ export interface NewAgentAction { sent_at?: string; } -export type AgentAction = NewAgentAction & { +export interface AgentAction extends NewAgentAction { id: string; agent_id: string; created_at: string; -} & SavedObjectAttributes; +} -export interface AgentActionSOAttributes extends NewAgentAction, SavedObjectAttributes { +export interface AgentActionSOAttributes extends SavedObjectAttributes { + type: 'CONFIG_CHANGE' | 'DATA_DUMP' | 'RESUME' | 'PAUSE'; + sent_at?: string; created_at: string; agent_id: string; + data?: string; } export interface AgentEvent { @@ -64,6 +67,7 @@ interface AgentBase { shared_id?: string; access_api_key_id?: string; default_api_key?: string; + default_api_key_id?: string; config_id?: string; config_revision?: number | null; config_newest_revision?: number; diff --git a/x-pack/plugins/ingest_manager/server/plugin.ts b/x-pack/plugins/ingest_manager/server/plugin.ts index 55b1d2961bca16..c07d2efba67567 100644 --- a/x-pack/plugins/ingest_manager/server/plugin.ts +++ b/x-pack/plugins/ingest_manager/server/plugin.ts @@ -13,7 +13,10 @@ import { SavedObjectsServiceStart, } from 'kibana/server'; import { LicensingPluginSetup } from '../../licensing/server'; -import { EncryptedSavedObjectsPluginStart } from '../../encrypted_saved_objects/server'; +import { + EncryptedSavedObjectsPluginStart, + EncryptedSavedObjectsPluginSetup, +} from '../../encrypted_saved_objects/server'; import { SecurityPluginSetup } from '../../security/server'; import { PluginSetupContract as FeaturesPluginSetup } from '../../features/server'; import { @@ -26,7 +29,7 @@ import { AGENT_EVENT_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, } from './constants'; - +import { registerEncryptedSavedObjects } from './saved_objects'; import { registerEPMRoutes, registerDatasourceRoutes, @@ -46,6 +49,7 @@ export interface IngestManagerSetupDeps { licensing: LicensingPluginSetup; security?: SecurityPluginSetup; features?: FeaturesPluginSetup; + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup; } export type IngestManagerStartDeps = object; @@ -97,6 +101,8 @@ export class IngestManagerPlugin this.security = deps.security; } + registerEncryptedSavedObjects(deps.encryptedSavedObjects); + // Register feature // TODO: Flesh out privileges if (deps.features) { diff --git a/x-pack/plugins/ingest_manager/server/saved_objects.ts b/x-pack/plugins/ingest_manager/server/saved_objects.ts index 0a7229b1f28072..37a00228443e15 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects.ts @@ -13,6 +13,7 @@ import { AGENT_ACTION_SAVED_OBJECT_TYPE, ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, } from './constants'; +import { EncryptedSavedObjectsPluginSetup } from '../../encrypted_saved_objects/server'; /* * Saved object mappings @@ -35,7 +36,7 @@ export const savedObjectMappings = { last_checkin: { type: 'date' }, config_revision: { type: 'integer' }, config_newest_revision: { type: 'integer' }, - // FIXME_INGEST https://github.com/elastic/kibana/issues/56554 + default_api_key_id: { type: 'keyword' }, default_api_key: { type: 'keyword' }, updated_at: { type: 'date' }, current_error_events: { type: 'text' }, @@ -45,8 +46,7 @@ export const savedObjectMappings = { properties: { agent_id: { type: 'keyword' }, type: { type: 'keyword' }, - // FIXME_INGEST https://github.com/elastic/kibana/issues/56554 - data: { type: 'flattened' }, + data: { type: 'binary' }, sent_at: { type: 'date' }, created_at: { type: 'date' }, }, @@ -83,7 +83,6 @@ export const savedObjectMappings = { properties: { name: { type: 'keyword' }, type: { type: 'keyword' }, - // FIXME_INGEST https://github.com/elastic/kibana/issues/56554 api_key: { type: 'binary' }, api_key_id: { type: 'keyword' }, config_id: { type: 'keyword' }, @@ -100,8 +99,6 @@ export const savedObjectMappings = { is_default: { type: 'boolean' }, hosts: { type: 'keyword' }, ca_sha256: { type: 'keyword' }, - // FIXME_INGEST https://github.com/elastic/kibana/issues/56554 - api_key: { type: 'keyword' }, fleet_enroll_username: { type: 'binary' }, fleet_enroll_password: { type: 'binary' }, config: { type: 'flattened' }, @@ -165,3 +162,61 @@ export const savedObjectMappings = { }, }, }; + +export function registerEncryptedSavedObjects( + encryptedSavedObjects: EncryptedSavedObjectsPluginSetup +) { + // Encrypted saved objects + encryptedSavedObjects.registerType({ + type: ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + attributesToEncrypt: new Set(['api_key']), + attributesToExcludeFromAAD: new Set([ + 'name', + 'type', + 'api_key_id', + 'config_id', + 'created_at', + 'updated_at', + 'expire_at', + 'active', + ]), + }); + encryptedSavedObjects.registerType({ + type: OUTPUT_SAVED_OBJECT_TYPE, + attributesToEncrypt: new Set(['fleet_enroll_username', 'fleet_enroll_password']), + attributesToExcludeFromAAD: new Set([ + 'name', + 'type', + 'is_default', + 'hosts', + 'ca_sha256', + 'config', + ]), + }); + encryptedSavedObjects.registerType({ + type: AGENT_SAVED_OBJECT_TYPE, + attributesToEncrypt: new Set(['default_api_key']), + attributesToExcludeFromAAD: new Set([ + 'shared_id', + 'type', + 'active', + 'enrolled_at', + 'access_api_key_id', + 'version', + 'user_provided_metadata', + 'local_metadata', + 'config_id', + 'last_updated', + 'last_checkin', + 'config_revision', + 'config_newest_revision', + 'updated_at', + 'current_error_events', + ]), + }); + encryptedSavedObjects.registerType({ + type: AGENT_ACTION_SAVED_OBJECT_TYPE, + attributesToEncrypt: new Set(['data']), + attributesToExcludeFromAAD: new Set(['agent_id', 'type', 'sent_at', 'created_at']), + }); +} diff --git a/x-pack/plugins/ingest_manager/server/services/agents/acks.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/acks.test.ts index b4c1f09015a69b..62871710d9877b 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/acks.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/acks.test.ts @@ -6,6 +6,8 @@ import Boom from 'boom'; import { SavedObjectsBulkResponse } from 'kibana/server'; import { savedObjectsClientMock } from '../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock'; +import { encryptedSavedObjectsMock } from '../../../../../plugins/encrypted_saved_objects/server/mocks'; + import { Agent, AgentAction, @@ -14,10 +16,31 @@ import { } from '../../../common/types/models'; import { AGENT_TYPE_PERMANENT } from '../../../common/constants'; import { acknowledgeAgentActions } from './acks'; +import { appContextService } from '../app_context'; +import { IngestManagerAppContext } from '../../plugin'; describe('test agent acks services', () => { it('should succeed on valid and matched actions', async () => { const mockSavedObjectsClient = savedObjectsClientMock.create(); + const mockStartEncryptedSOClient = encryptedSavedObjectsMock.createStart(); + appContextService.start(({ + encryptedSavedObjects: mockStartEncryptedSOClient, + } as unknown) as IngestManagerAppContext); + + mockStartEncryptedSOClient.getDecryptedAsInternalUser.mockReturnValue( + Promise.resolve({ + id: 'action1', + references: [], + type: 'agent_actions', + attributes: { + type: 'CONFIG_CHANGE', + agent_id: 'id', + sent_at: '2020-03-14T19:45:02.620Z', + timestamp: '2019-01-04T14:32:03.36764-05:00', + created_at: '2020-03-14T19:45:02.620Z', + }, + }) + ); mockSavedObjectsClient.bulkGet.mockReturnValue( Promise.resolve({ diff --git a/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts index f2e671c6dbaa87..29143502247aa0 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/actions.test.ts @@ -6,17 +6,17 @@ import { createAgentAction } from './actions'; import { SavedObject } from 'kibana/server'; -import { AgentAction, AgentActionSOAttributes } from '../../../common/types/models'; +import { AgentAction } from '../../../common/types/models'; import { savedObjectsClientMock } from '../../../../../../src/core/server/saved_objects/service/saved_objects_client.mock'; describe('test agent actions services', () => { it('should create a new action', async () => { const mockSavedObjectsClient = savedObjectsClientMock.create(); - const newAgentAction: AgentActionSOAttributes = { + const newAgentAction: Omit = { agent_id: 'agentid', type: 'CONFIG_CHANGE', - data: 'data', + data: { content: 'data' }, sent_at: '2020-03-14T19:45:02.620Z', created_at: '2020-03-14T19:45:02.620Z', }; @@ -31,7 +31,7 @@ describe('test agent actions services', () => { .calls[0][1] as unknown) as AgentAction; expect(createdAction).toBeDefined(); expect(createdAction?.type).toEqual(newAgentAction.type); - expect(createdAction?.data).toEqual(newAgentAction.data); + expect(createdAction?.data).toEqual(JSON.stringify(newAgentAction.data)); expect(createdAction?.sent_at).toEqual(newAgentAction.sent_at); }); }); diff --git a/x-pack/plugins/ingest_manager/server/services/agents/actions.ts b/x-pack/plugins/ingest_manager/server/services/agents/actions.ts index a8ef0820f8d9f3..1bb177e54282d0 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/actions.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/actions.ts @@ -8,16 +8,21 @@ import { SavedObjectsClientContract } from 'kibana/server'; import { Agent, AgentAction, AgentActionSOAttributes } from '../../../common/types/models'; import { AGENT_ACTION_SAVED_OBJECT_TYPE } from '../../../common/constants'; import { savedObjectToAgentAction } from './saved_objects'; +import { appContextService } from '../app_context'; export async function createAgentAction( soClient: SavedObjectsClientContract, - newAgentAction: AgentActionSOAttributes + newAgentAction: Omit ): Promise { const so = await soClient.create(AGENT_ACTION_SAVED_OBJECT_TYPE, { ...newAgentAction, + data: newAgentAction.data ? JSON.stringify(newAgentAction.data) : undefined, }); - return savedObjectToAgentAction(so); + const agentAction = savedObjectToAgentAction(so); + agentAction.data = newAgentAction.data; + + return agentAction; } export async function getAgentActionsForCheckin( @@ -29,21 +34,47 @@ export async function getAgentActionsForCheckin( filter: `not ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.sent_at: * and ${AGENT_ACTION_SAVED_OBJECT_TYPE}.attributes.agent_id:${agentId}`, }); - return res.saved_objects.map(savedObjectToAgentAction); + return Promise.all( + res.saved_objects.map(async so => { + // Get decrypted actions + return savedObjectToAgentAction( + await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser( + AGENT_ACTION_SAVED_OBJECT_TYPE, + so.id + ) + ); + }) + ); } export async function getAgentActionByIds( soClient: SavedObjectsClientContract, actionIds: string[] ) { - const res = await soClient.bulkGet( - actionIds.map(actionId => ({ - id: actionId, - type: AGENT_ACTION_SAVED_OBJECT_TYPE, - })) - ); + const actions = ( + await soClient.bulkGet( + actionIds.map(actionId => ({ + id: actionId, + type: AGENT_ACTION_SAVED_OBJECT_TYPE, + })) + ) + ).saved_objects.map(savedObjectToAgentAction); - return res.saved_objects.map(savedObjectToAgentAction); + return Promise.all( + actions.map(async action => { + // Get decrypted actions + return savedObjectToAgentAction( + await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser( + AGENT_ACTION_SAVED_OBJECT_TYPE, + action.id + ) + ); + }) + ); } export interface ActionsService { diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts index ec10ca6e77e05f..72a86d7c8158e0 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin.test.ts @@ -53,12 +53,12 @@ describe('Agent checkin service', () => { agent_id: 'agent1', type: 'CONFIG_CHANGE', created_at: new Date().toISOString(), - data: JSON.stringify({ + data: { config: { id: 'config1', revision: 2, }, - }), + }, }, ] ); @@ -80,24 +80,24 @@ describe('Agent checkin service', () => { agent_id: 'agent1', type: 'CONFIG_CHANGE', created_at: new Date().toISOString(), - data: JSON.stringify({ + data: { config: { id: 'config2', revision: 2, }, - }), + }, }, { id: 'action1', agent_id: 'agent1', type: 'CONFIG_CHANGE', created_at: new Date().toISOString(), - data: JSON.stringify({ + data: { config: { id: 'config1', revision: 1, }, - }), + }, }, ] ); diff --git a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts index 2873aad7f691aa..c96a81ed9b7587 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/checkin.ts @@ -17,6 +17,7 @@ import { agentConfigService } from '../agent_config'; import * as APIKeysService from '../api_keys'; import { AGENT_SAVED_OBJECT_TYPE, AGENT_EVENT_SAVED_OBJECT_TYPE } from '../../constants'; import { getAgentActionsForCheckin, createAgentAction } from './actions'; +import { appContextService } from '../app_context'; export async function agentCheckin( soClient: SavedObjectsClientContract, @@ -27,7 +28,6 @@ export async function agentCheckin( const updateData: { last_checkin: string; default_api_key?: string; - actions?: AgentAction[]; local_metadata?: string; current_error_events?: string; } = { @@ -38,11 +38,17 @@ export async function agentCheckin( // Generate new agent config if config is updated if (agent.config_id && shouldCreateConfigAction(agent, actions)) { + const { + attributes: { default_api_key: defaultApiKey }, + } = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser(AGENT_SAVED_OBJECT_TYPE, agent.id); + const config = await agentConfigService.getFullConfig(soClient, agent.config_id); if (config) { // Assign output API keys // We currently only support default ouput - if (!agent.default_api_key) { + if (!defaultApiKey) { updateData.default_api_key = await APIKeysService.generateOutputApiKey( soClient, 'default', @@ -50,7 +56,7 @@ export async function agentCheckin( ); } // Mutate the config to set the api token for this agent - config.outputs.default.api_key = agent.default_api_key || updateData.default_api_key; + config.outputs.default.api_key = defaultApiKey || updateData.default_api_key; const configChangeAction = await createAgentAction(soClient, { agent_id: agent.id, @@ -62,9 +68,6 @@ export async function agentCheckin( actions.push(configChangeAction); } } - if (localMetadata) { - updateData.local_metadata = JSON.stringify(localMetadata); - } const { updatedErrorEvents } = await processEventsForCheckin(soClient, agent, events); @@ -172,7 +175,7 @@ export function shouldCreateConfigAction(agent: Agent, actions: AgentAction[]): return false; } - const data = JSON.parse(action.data); + const { data } = action; return ( data.config.id === agent.config_id && data.config.revision === agent.config_newest_revision diff --git a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts index aa885207406872..b182662e0fb4e2 100644 --- a/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts +++ b/x-pack/plugins/ingest_manager/server/services/agents/saved_objects.ts @@ -38,5 +38,6 @@ export function savedObjectToAgentAction(so: SavedObject(AGENT_SAVED_OBJECT_TYPE, agentId, { diff --git a/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts b/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts index a6a2db8be4e9d9..c9ead09b0908d5 100644 --- a/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts +++ b/x-pack/plugins/ingest_manager/server/services/api_keys/enrollment_api_key.ts @@ -10,6 +10,7 @@ import { EnrollmentAPIKey, EnrollmentAPIKeySOAttributes } from '../../types'; import { ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE } from '../../constants'; import { createAPIKey, invalidateAPIKey } from './security'; import { agentConfigService } from '../agent_config'; +import { appContextService } from '../app_context'; export async function listEnrollmentApiKeys( soClient: SavedObjectsClientContract, @@ -45,9 +46,13 @@ export async function listEnrollmentApiKeys( } export async function getEnrollmentAPIKey(soClient: SavedObjectsClientContract, id: string) { - return savedObjectToEnrollmentApiKey( - await soClient.get(ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, id) - ); + const so = await appContextService + .getEncryptedSavedObjects() + .getDecryptedAsInternalUser( + ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + id + ); + return savedObjectToEnrollmentApiKey(so); } /** @@ -120,16 +125,19 @@ export async function generateEnrollmentAPIKey( const apiKey = Buffer.from(`${key.id}:${key.api_key}`).toString('base64'); - return savedObjectToEnrollmentApiKey( - await soClient.create(ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, { + const so = await soClient.create( + ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE, + { active: true, api_key_id: key.id, api_key: apiKey, name, config_id: configId, created_at: new Date().toISOString(), - }) + } ); + + return getEnrollmentAPIKey(soClient, so.id); } function savedObjectToEnrollmentApiKey({ diff --git a/x-pack/plugins/ingest_manager/server/services/app_context.ts b/x-pack/plugins/ingest_manager/server/services/app_context.ts index a0a7c8dd7c05a0..e917d2edd13090 100644 --- a/x-pack/plugins/ingest_manager/server/services/app_context.ts +++ b/x-pack/plugins/ingest_manager/server/services/app_context.ts @@ -34,6 +34,9 @@ class AppContextService { public stop() {} public getEncryptedSavedObjects() { + if (!this.encryptedSavedObjects) { + throw new Error('Encrypted saved object start service not set.'); + } return this.encryptedSavedObjects; } diff --git a/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts b/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts index 2acfca63995f1c..d33b92acf95a54 100644 --- a/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts +++ b/x-pack/test/api_integration/apis/fleet/unenroll_agent.ts @@ -44,6 +44,7 @@ export default function(providerContext: FtrProviderContext) { }); // @ts-ignore agentDoc.agents.access_api_key_id = accessAPIKeyId; + agentDoc.agents.default_api_key_id = outputAPIKeyBody.id; agentDoc.agents.default_api_key = Buffer.from( `${outputAPIKeyBody.id}:${outputAPIKeyBody.api_key}` ).toString('base64');