From fbcd6dbf90d3531f37ac4ba4038774d73679b42b Mon Sep 17 00:00:00 2001 From: Ignacio Rivas Date: Fri, 21 Apr 2023 15:47:28 +0200 Subject: [PATCH] [Ingest Pipelines] Add attachment processor (#155226) --- .../__jest__/processors/attachment.test.tsx | 122 +++++++++++ .../__jest__/processors/processor.helpers.tsx | 5 + .../processor_form/processors/attachment.tsx | 202 ++++++++++++++++++ .../processor_form/processors/index.ts | 1 + .../shared/map_processor_type_to_form.tsx | 18 ++ 5 files changed, 348 insertions(+) create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/attachment.test.tsx create mode 100644 x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/attachment.tsx diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/attachment.test.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/attachment.test.tsx new file mode 100644 index 000000000000000..b4cfbf3046db0b9 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/attachment.test.tsx @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { act } from 'react-dom/test-utils'; +import { setup, SetupResult, getProcessorValue, setupEnvironment } from './processor.helpers'; + +const ATTACHMENT_TYPE = 'attachment'; + +describe('Processor: Attachment', () => { + let onUpdate: jest.Mock; + let testBed: SetupResult; + const { httpSetup } = setupEnvironment(); + + beforeAll(() => { + jest.useFakeTimers({ legacyFakeTimers: true }); + }); + + afterAll(() => { + jest.useRealTimers(); + }); + + beforeEach(async () => { + onUpdate = jest.fn(); + + await act(async () => { + testBed = await setup(httpSetup, { + value: { + processors: [], + }, + onFlyoutOpen: jest.fn(), + onUpdate, + }); + }); + + const { component, actions } = testBed; + + component.update(); + + // Open flyout to add new processor + actions.addProcessor(); + // Add type (the other fields are not visible until a type is selected) + await actions.addProcessorType(ATTACHMENT_TYPE); + }); + + test('prevents form submission if required fields are not provided', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Click submit button with only the type defined + await saveNewProcessor(); + + // Expect form error as "field" is a required parameter + expect(form.getErrorsMessages()).toEqual([ + 'A field value is required.', // "Field" input + ]); + }); + + test('saves with default parameter values', async () => { + const { + actions: { saveNewProcessor }, + form, + } = testBed; + + // Add "field" value + form.setInputValue('fieldNameField.input', 'test_attachment_processor'); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, ATTACHMENT_TYPE); + + expect(processors[0][ATTACHMENT_TYPE]).toEqual({ + field: 'test_attachment_processor', + }); + }); + + test('saves with optional parameter values', async () => { + const { + actions: { saveNewProcessor }, + form, + find, + component, + } = testBed; + + // Add required fields + form.setInputValue('fieldNameField.input', 'test_attachment_processor'); + + // Add optional fields + form.setInputValue('targetField.input', 'test_target'); + form.setInputValue('indexedCharsField.input', '123456'); + form.setInputValue('indexedCharsFieldField.input', 'indexed_chars_field'); + form.toggleEuiSwitch('removeBinaryField.input'); + form.setInputValue('resourceNameField.input', 'resource_name_field'); + + // Add "networkDirectionField" value (required) + await act(async () => { + find('propertiesField').simulate('change', [{ label: 'content' }]); + }); + component.update(); + + // Save the field + await saveNewProcessor(); + + const processors = getProcessorValue(onUpdate, ATTACHMENT_TYPE); + + expect(processors[0][ATTACHMENT_TYPE]).toEqual({ + field: 'test_attachment_processor', + target_field: 'test_target', + properties: ['content'], + indexed_chars: '123456', + indexed_chars_field: 'indexed_chars_field', + remove_binary: true, + resource_name: 'resource_name_field', + }); + }); +}); diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx index 43ebf84f6ef30a6..125d8758ca0d0b0 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/__jest__/processors/processor.helpers.tsx @@ -190,6 +190,11 @@ type TestSubject = | 'droppableList.input-2' | 'prefixField.input' | 'suffixField.input' + | 'indexedCharsField.input' + | 'indexedCharsFieldField.input' + | 'removeBinaryField.input' + | 'resourceNameField.input' + | 'propertiesField' | 'tileTypeField' | 'targetFormatField' | 'parentField.input' diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/attachment.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/attachment.tsx new file mode 100644 index 000000000000000..57f26bdaf204f35 --- /dev/null +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/attachment.tsx @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { FunctionComponent } from 'react'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n-react'; +import { EuiCode, EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; + +import { + ComboBoxField, + FIELD_TYPES, + UseField, + ToggleField, + Field, +} from '../../../../../../shared_imports'; + +import { FieldNameField } from './common_fields/field_name_field'; +import { TargetField } from './common_fields/target_field'; +import { IgnoreMissingField } from './common_fields/ignore_missing_field'; +import { FieldsConfig, to, from } from './shared'; + +const propertyValues: string[] = [ + 'content', + 'title', + 'author', + 'keywords', + 'date', + 'content_type', + 'content_length', + 'language', +]; + +const fieldsConfig: FieldsConfig = { + /* Optional field configs */ + indexed_chars: { + type: FIELD_TYPES.NUMBER, + serializer: from.emptyStringToUndefined, + label: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.attachment.indexedCharsFieldLabel', + { + defaultMessage: 'Indexed chars (optional)', + } + ), + helpText: ( + {'100000'} }} + /> + ), + }, + indexed_chars_field: { + type: FIELD_TYPES.TEXT, + serializer: from.emptyStringToUndefined, + label: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.attachment.indexedCharsFieldFieldLabel', + { + defaultMessage: 'Indexed chars field (optional)', + } + ), + helpText: ( + {'null'} }} + /> + ), + }, + properties: { + type: FIELD_TYPES.COMBO_BOX, + deserializer: to.arrayOfStrings, + serializer: from.optionalArrayOfStrings, + label: i18n.translate('xpack.ingestPipelines.pipelineEditor.attachment.propertiesFieldLabel', { + defaultMessage: 'Properties (optional)', + }), + helpText: ( + {'all'} }} + /> + ), + }, + remove_binary: { + type: FIELD_TYPES.TOGGLE, + defaultValue: false, + deserializer: to.booleanOrUndef, + serializer: from.undefinedIfValue(false), + label: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.attachment.removeBinaryFieldLabel', + { + defaultMessage: 'Remove binary', + } + ), + helpText: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.attachment.removeBinaryFieldHelpText', + { + defaultMessage: 'If enabled, the binary field will be removed from the document.', + } + ), + }, + resource_name: { + type: FIELD_TYPES.TEXT, + serializer: from.emptyStringToUndefined, + label: i18n.translate( + 'xpack.ingestPipelines.pipelineEditor.attachment.resourceNameFieldLabel', + { + defaultMessage: 'Resource name (optional)', + } + ), + helpText: ( + + ), + }, +}; + +export const Attachment: FunctionComponent = () => { + return ( + <> + + + + } + /> + + + {'attachment'} }} + /> + } + /> + + + + + + + + + + + + + + + + + ({ label })), + }} + path="fields.properties" + /> + + + + + + + + ); +}; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/index.ts b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/index.ts index 3512dafdd2854ff..210d113bd2aba53 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/index.ts +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/processor_form/processors/index.ts @@ -8,6 +8,7 @@ // please try to keep this list sorted by module name (e.g. './bar' before './foo') export { Append } from './append'; +export { Attachment } from './attachment'; export { Bytes } from './bytes'; export { Circle } from './circle'; export { CommunityId } from './community_id'; diff --git a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/shared/map_processor_type_to_form.tsx b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/shared/map_processor_type_to_form.tsx index 75bbd764097ad78..60f66dbb415f359 100644 --- a/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/shared/map_processor_type_to_form.tsx +++ b/x-pack/plugins/ingest_pipelines/public/application/components/pipeline_editor/components/shared/map_processor_type_to_form.tsx @@ -14,6 +14,7 @@ import { LicenseType } from '../../../../../types'; import { Append, + Attachment, Bytes, Circle, CommunityId, @@ -100,6 +101,23 @@ export const mapProcessorTypeToDescriptor: MapProcessorTypeToDescriptor = { }, }), }, + attachment: { + FieldsComponent: Attachment, + docLinkPath: '/attachment.html', + label: i18n.translate('xpack.ingestPipelines.processors.label.attachment', { + defaultMessage: 'Attachment', + }), + typeDescription: i18n.translate('xpack.ingestPipelines.processors.description.attachment', { + defaultMessage: 'Extract file attachments in common formats (such as PPT, XLS, and PDF).', + }), + getDefaultDescription: ({ field }) => + i18n.translate('xpack.ingestPipelines.processors.defaultDescription.attachment', { + defaultMessage: 'Extracts attachment from "{field}"', + values: { + field, + }, + }), + }, bytes: { FieldsComponent: Bytes, docLinkPath: '/bytes-processor.html',