+
diff --git a/x-pack/legacy/plugins/maps/public/angular/map_controller.js b/x-pack/legacy/plugins/maps/public/angular/map_controller.js
index 41c618d68a68e40..b9354dd0a0dddd0 100644
--- a/x-pack/legacy/plugins/maps/public/angular/map_controller.js
+++ b/x-pack/legacy/plugins/maps/public/angular/map_controller.js
@@ -66,6 +66,7 @@ const app = uiModules.get(MAP_APP_PATH, []);
app.controller('GisMapController', ($scope, $route, kbnUrl, localStorage, AppState, globalState) => {
const { filterManager } = npStart.plugins.data.query;
const savedMap = $route.current.locals.map;
+ $scope.screenTitle = savedMap.title;
let unsubscribe;
let initialLayerListConfig;
const $state = new AppState();
diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js
index ef2819f1f372c67..4b04251edd94a01 100644
--- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js
+++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.js
@@ -11,7 +11,6 @@ import {
DECIMAL_DEGREES_PRECISION,
ES_GEO_FIELD_TYPE,
ES_SPATIAL_RELATIONS,
- FEATURE_ID_PROPERTY_NAME,
GEO_JSON_TYPE,
POLYGON_COORDINATES_EXTERIOR_INDEX,
LON_INDEX,
@@ -81,12 +80,10 @@ export function hitsToGeoJson(hits, flattenHit, geoFieldName, geoFieldType) {
features.push({
type: 'Feature',
geometry: tmpGeometriesAccumulator[j],
- properties: {
- ...properties,
- // _id is not unique across Kibana index pattern. Multiple ES indices could have _id collisions
- // Need to prefix with _index to guarantee uniqueness
- [FEATURE_ID_PROPERTY_NAME]: `${properties._index}:${properties._id}:${j}`
- }
+ // _id is not unique across Kibana index pattern. Multiple ES indices could have _id collisions
+ // Need to prefix with _index to guarantee uniqueness
+ id: `${properties._index}:${properties._id}:${j}`,
+ properties,
});
}
}
diff --git a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js
index 0b84b4c32f4ac2b..45aa2af15eb9d50 100644
--- a/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js
+++ b/x-pack/legacy/plugins/maps/public/elasticsearch_geo_utils.test.js
@@ -74,8 +74,8 @@ describe('hitsToGeoJson', () => {
coordinates: [100, 20],
type: 'Point',
},
+ id: 'index1:doc1:0',
properties: {
- __kbn__feature_id__: 'index1:doc1:0',
_id: 'doc1',
_index: 'index1',
},
@@ -139,8 +139,8 @@ describe('hitsToGeoJson', () => {
coordinates: [100, 20],
type: 'Point',
},
+ id: 'index1:doc1:0',
properties: {
- __kbn__feature_id__: 'index1:doc1:0',
_id: 'doc1',
_index: 'index1',
myField: 8
@@ -152,8 +152,8 @@ describe('hitsToGeoJson', () => {
coordinates: [110, 30],
type: 'Point',
},
+ id: 'index1:doc1:1',
properties: {
- __kbn__feature_id__: 'index1:doc1:1',
_id: 'doc1',
_index: 'index1',
myField: 8
diff --git a/x-pack/legacy/plugins/maps/public/kibana_services.js b/x-pack/legacy/plugins/maps/public/kibana_services.js
index 12fab24d1f8d65f..71690145427106a 100644
--- a/x-pack/legacy/plugins/maps/public/kibana_services.js
+++ b/x-pack/legacy/plugins/maps/public/kibana_services.js
@@ -4,13 +4,13 @@
* you may not use this file except in compliance with the Elastic License.
*/
-import { getRequestInspectorStats, getResponseInspectorStats } from 'ui/courier/utils/courier_inspector_utils';
+import { getRequestInspectorStats, getResponseInspectorStats } from '../../../../../src/legacy/ui/public/courier';
export { xpackInfo } from 'plugins/xpack_main/services/xpack_info';
import { start as data } from '../../../../../src/legacy/core_plugins/data/public/legacy';
import { esFilters } from '../../../../../src/plugins/data/public';
export const SPATIAL_FILTER_TYPE = esFilters.FILTERS.SPATIAL_FILTER;
-export { SearchSource } from 'ui/courier';
+export { SearchSource } from '../../../../../src/legacy/ui/public/courier';
export const indexPatternService = data.indexPatterns.indexPatterns;
export async function fetchSearchSourceAndRecordWithInspector({
diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js
index 11a02d58a919800..920253d15eaee12 100644
--- a/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js
+++ b/x-pack/legacy/plugins/maps/public/layers/sources/client_file_source/geojson_file_source.js
@@ -10,7 +10,6 @@ import {
ES_GEO_FIELD_TYPE,
GEOJSON_FILE,
ES_SIZE_LIMIT,
- FEATURE_ID_PROPERTY_NAME
} from '../../../../common/constants';
import { ClientFileCreateSourceEditor } from './create_client_file_source_editor';
import { ESSearchSource } from '../es_search_source';
@@ -137,21 +136,8 @@ export class GeojsonFileSource extends AbstractVectorSource {
}
async getGeoJsonWithMeta() {
- const copiedPropsFeatures = this._descriptor.__featureCollection.features.map((feature, index) => {
- const properties = feature.properties ? { ...feature.properties } : {};
- properties[FEATURE_ID_PROPERTY_NAME] = index;
- return {
- type: 'Feature',
- geometry: feature.geometry,
- properties,
- };
- });
-
return {
- data: {
- type: 'FeatureCollection',
- features: copiedPropsFeatures
- },
+ data: this._descriptor.__featureCollection,
meta: {}
};
}
diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/ems_file_source/ems_file_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/ems_file_source/ems_file_source.js
index b2e04f56e571810..fcd52683b70fffd 100644
--- a/x-pack/legacy/plugins/maps/public/layers/sources/ems_file_source/ems_file_source.js
+++ b/x-pack/legacy/plugins/maps/public/layers/sources/ems_file_source/ems_file_source.js
@@ -7,7 +7,7 @@
import { AbstractVectorSource } from '../vector_source';
import { VECTOR_SHAPE_TYPES } from '../vector_feature_types';
import React from 'react';
-import { EMS_FILE, FEATURE_ID_PROPERTY_NAME, FIELD_ORIGIN } from '../../../../common/constants';
+import { EMS_FILE, FIELD_ORIGIN } from '../../../../common/constants';
import { getEMSClient } from '../../../meta';
import { EMSFileCreateSourceEditor } from './create_source_editor';
import { i18n } from '@kbn/i18n';
@@ -94,7 +94,7 @@ export class EMSFileSource extends AbstractVectorSource {
return field.type === 'id';
});
featureCollection.features.forEach((feature, index) => {
- feature.properties[FEATURE_ID_PROPERTY_NAME] = emsIdField
+ feature.id = emsIdField
? feature.properties[emsIdField.id]
: index;
});
diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js
index c83f12ce992ff05..d26bfd8bbeacb36 100644
--- a/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js
+++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_geo_grid_source/convert_to_geojson.js
@@ -6,7 +6,7 @@
import { RENDER_AS } from './render_as';
import { getTileBoundingBox } from './geo_tile_utils';
-import { EMPTY_FEATURE_COLLECTION, FEATURE_ID_PROPERTY_NAME } from '../../../../common/constants';
+import { EMPTY_FEATURE_COLLECTION } from '../../../../common/constants';
export function convertToGeoJson({ table, renderAs }) {
@@ -34,9 +34,7 @@ export function convertToGeoJson({ table, renderAs }) {
return;
}
- const properties = {
- [FEATURE_ID_PROPERTY_NAME]: gridKey
- };
+ const properties = {};
metricColumns.forEach(metricColumn => {
properties[metricColumn.aggConfig.id] = row[metricColumn.id];
});
@@ -49,6 +47,7 @@ export function convertToGeoJson({ table, renderAs }) {
geocentroidColumn,
renderAs,
}),
+ id: gridKey,
properties
});
});
diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js
index c334776e6c4e87d..ae9435dc42c69c3 100644
--- a/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js
+++ b/x-pack/legacy/plugins/maps/public/layers/sources/es_pew_pew_source/convert_to_lines.js
@@ -6,8 +6,6 @@
import _ from 'lodash';
-import { FEATURE_ID_PROPERTY_NAME } from '../../../../common/constants';
-
const LAT_INDEX = 0;
const LON_INDEX = 1;
@@ -47,10 +45,10 @@ export function convertToLines(esResponse) {
type: 'LineString',
coordinates: [[sourceCentroid.location.lon, sourceCentroid.location.lat], dest]
},
+ id: `${dest.join()},${key}`,
properties: {
- [FEATURE_ID_PROPERTY_NAME]: `${dest.join()},${key}`,
...rest
- }
+ },
});
}
}
diff --git a/x-pack/legacy/plugins/maps/public/layers/sources/kibana_regionmap_source/kibana_regionmap_source.js b/x-pack/legacy/plugins/maps/public/layers/sources/kibana_regionmap_source/kibana_regionmap_source.js
index ffccb18a6919262..e29887edcf7d9fb 100644
--- a/x-pack/legacy/plugins/maps/public/layers/sources/kibana_regionmap_source/kibana_regionmap_source.js
+++ b/x-pack/legacy/plugins/maps/public/layers/sources/kibana_regionmap_source/kibana_regionmap_source.js
@@ -10,7 +10,7 @@ import { CreateSourceEditor } from './create_source_editor';
import { getKibanaRegionList } from '../../../meta';
import { i18n } from '@kbn/i18n';
import { getDataSourceLabel } from '../../../../common/i18n_getters';
-import { FEATURE_ID_PROPERTY_NAME, FIELD_ORIGIN } from '../../../../common/constants';
+import { FIELD_ORIGIN } from '../../../../common/constants';
import { KibanaRegionField } from '../../fields/kibana_region_field';
export class KibanaRegionmapSource extends AbstractVectorSource {
@@ -91,9 +91,6 @@ export class KibanaRegionmapSource extends AbstractVectorSource {
featureCollectionPath: vectorFileMeta.meta.feature_collection_path,
fetchUrl: vectorFileMeta.url
});
- featureCollection.features.forEach((feature, index) => {
- feature.properties[FEATURE_ID_PROPERTY_NAME] = index;
- });
return {
data: featureCollection
};
diff --git a/x-pack/legacy/plugins/maps/public/layers/util/assign_feature_ids.js b/x-pack/legacy/plugins/maps/public/layers/util/assign_feature_ids.js
new file mode 100644
index 000000000000000..2c0d08f86cfc0ea
--- /dev/null
+++ b/x-pack/legacy/plugins/maps/public/layers/util/assign_feature_ids.js
@@ -0,0 +1,57 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import _ from 'lodash';
+import { FEATURE_ID_PROPERTY_NAME } from '../../../common/constants';
+
+let idCounter = 0;
+
+function generateNumericalId() {
+ const newId = idCounter < Number.MAX_SAFE_INTEGER ? idCounter : 0;
+ idCounter = newId + 1;
+ return newId;
+}
+
+export function assignFeatureIds(featureCollection) {
+
+ // wrt https://github.com/elastic/kibana/issues/39317
+ // In constrained resource environments, mapbox-gl may throw a stackoverflow error due to hitting the browser's recursion limit. This crashes Kibana.
+ // This error is thrown in mapbox-gl's quicksort implementation, when it is sorting all the features by id.
+ // This is a work-around to avoid hitting such a worst-case
+ // This was tested as a suitable work-around for mapbox-gl 0.54
+ // The core issue itself is likely related to https://github.com/mapbox/mapbox-gl-js/issues/6086
+
+ // This only shuffles the id-assignment, _not_ the features in the collection
+ // The reason for this is that we do not want to modify the feature-ordering, which is the responsiblity of the VectorSource#.
+ const ids = [];
+ for (let i = 0; i < featureCollection.features.length; i++) {
+ const id = generateNumericalId();
+ ids.push(id);
+ }
+
+ const randomizedIds = _.shuffle(ids);
+ const features = [];
+ for (let i = 0; i < featureCollection.features.length; i++) {
+ const numericId = randomizedIds[i];
+ const feature = featureCollection.features[i];
+ features.push({
+ type: 'Feature',
+ geometry: feature.geometry, // do not copy geometry, this object can be massive
+ properties: {
+ // preserve feature id provided by source so features can be referenced across fetches
+ [FEATURE_ID_PROPERTY_NAME]: feature.id == null ? numericId : feature.id,
+ // create new object for properties so original is not polluted with kibana internal props
+ ...feature.properties,
+ },
+ id: numericId, // Mapbox feature state id, must be integer
+ });
+ }
+
+ return {
+ type: 'FeatureCollection',
+ features
+ };
+}
diff --git a/x-pack/legacy/plugins/maps/public/layers/util/assign_feature_ids.test.js b/x-pack/legacy/plugins/maps/public/layers/util/assign_feature_ids.test.js
new file mode 100644
index 000000000000000..0678070f568a21e
--- /dev/null
+++ b/x-pack/legacy/plugins/maps/public/layers/util/assign_feature_ids.test.js
@@ -0,0 +1,83 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { assignFeatureIds } from './assign_feature_ids';
+import { FEATURE_ID_PROPERTY_NAME } from '../../../common/constants';
+
+const featureId = 'myFeature1';
+
+test('should provide unique id when feature.id is not provided', () => {
+ const featureCollection = {
+ features: [
+ {
+ properties: {}
+ },
+ {
+ properties: {}
+ },
+ ]
+ };
+
+ const updatedFeatureCollection = assignFeatureIds(featureCollection);
+ const feature1 = updatedFeatureCollection.features[0];
+ const feature2 = updatedFeatureCollection.features[1];
+ expect(typeof feature1.id).toBe('number');
+ expect(typeof feature2.id).toBe('number');
+ expect(feature1.id).toBe(feature1.properties[FEATURE_ID_PROPERTY_NAME]);
+ expect(feature1.id).not.toBe(feature2.id);
+});
+
+test('should preserve feature id when provided', () => {
+ const featureCollection = {
+ features: [
+ {
+ id: featureId,
+ properties: {}
+ }
+ ]
+ };
+
+ const updatedFeatureCollection = assignFeatureIds(featureCollection);
+ const feature1 = updatedFeatureCollection.features[0];
+ expect(typeof feature1.id).toBe('number');
+ expect(feature1.id).not.toBe(feature1.properties[FEATURE_ID_PROPERTY_NAME]);
+ expect(feature1.properties[FEATURE_ID_PROPERTY_NAME]).toBe(featureId);
+});
+
+test('should preserve feature id for falsy value', () => {
+ const featureCollection = {
+ features: [
+ {
+ id: 0,
+ properties: {}
+ }
+ ]
+ };
+
+ const updatedFeatureCollection = assignFeatureIds(featureCollection);
+ const feature1 = updatedFeatureCollection.features[0];
+ expect(typeof feature1.id).toBe('number');
+ expect(feature1.id).not.toBe(feature1.properties[FEATURE_ID_PROPERTY_NAME]);
+ expect(feature1.properties[FEATURE_ID_PROPERTY_NAME]).toBe(0);
+});
+
+test('should not modify original feature properties', () => {
+ const featureProperties = {};
+ const featureCollection = {
+ features: [
+ {
+ id: featureId,
+ properties: featureProperties
+ }
+ ]
+ };
+
+ const updatedFeatureCollection = assignFeatureIds(featureCollection);
+ const feature1 = updatedFeatureCollection.features[0];
+ expect(feature1.properties[FEATURE_ID_PROPERTY_NAME]).toBe(featureId);
+ expect(featureProperties).not.toHaveProperty(FEATURE_ID_PROPERTY_NAME);
+});
+
diff --git a/x-pack/legacy/plugins/maps/public/layers/util/mb_filter_expressions.js b/x-pack/legacy/plugins/maps/public/layers/util/mb_filter_expressions.js
new file mode 100644
index 000000000000000..393c290d6966824
--- /dev/null
+++ b/x-pack/legacy/plugins/maps/public/layers/util/mb_filter_expressions.js
@@ -0,0 +1,62 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { GEO_JSON_TYPE, FEATURE_VISIBLE_PROPERTY_NAME } from '../../../common/constants';
+
+const VISIBILITY_FILTER_CLAUSE = ['all',
+ [
+ '==',
+ ['get', FEATURE_VISIBLE_PROPERTY_NAME],
+ true
+ ]
+];
+
+const CLOSED_SHAPE_MB_FILTER = [
+ 'any',
+ ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON],
+ ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON]
+];
+
+const VISIBLE_CLOSED_SHAPE_MB_FILTER = [
+ ...VISIBILITY_FILTER_CLAUSE,
+ CLOSED_SHAPE_MB_FILTER,
+];
+
+const ALL_SHAPE_MB_FILTER = [
+ 'any',
+ ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON],
+ ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON],
+ ['==', ['geometry-type'], GEO_JSON_TYPE.LINE_STRING],
+ ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_LINE_STRING]
+];
+
+const VISIBLE_ALL_SHAPE_MB_FILTER = [
+ ...VISIBILITY_FILTER_CLAUSE,
+ ALL_SHAPE_MB_FILTER,
+];
+
+const POINT_MB_FILTER = [
+ 'any',
+ ['==', ['geometry-type'], GEO_JSON_TYPE.POINT],
+ ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POINT]
+];
+
+const VISIBLE_POINT_MB_FILTER = [
+ ...VISIBILITY_FILTER_CLAUSE,
+ POINT_MB_FILTER,
+];
+
+export function getFillFilterExpression(hasJoins) {
+ return hasJoins ? VISIBLE_CLOSED_SHAPE_MB_FILTER : CLOSED_SHAPE_MB_FILTER;
+}
+
+export function getLineFilterExpression(hasJoins) {
+ return hasJoins ? VISIBLE_ALL_SHAPE_MB_FILTER : ALL_SHAPE_MB_FILTER;
+}
+
+export function getPointFilterExpression(hasJoins) {
+ return hasJoins ? VISIBLE_POINT_MB_FILTER : POINT_MB_FILTER;
+}
diff --git a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js
index 362c7bfd7254079..9b553803606ed4b 100644
--- a/x-pack/legacy/plugins/maps/public/layers/vector_layer.js
+++ b/x-pack/legacy/plugins/maps/public/layers/vector_layer.js
@@ -10,7 +10,6 @@ import { AbstractLayer } from './layer';
import { VectorStyle } from './styles/vector/vector_style';
import { InnerJoin } from './joins/inner_join';
import {
- GEO_JSON_TYPE,
FEATURE_ID_PROPERTY_NAME,
SOURCE_DATA_ID_ORIGIN,
FEATURE_VISIBLE_PROPERTY_NAME,
@@ -23,51 +22,12 @@ import { isRefreshOnlyQuery } from './util/is_refresh_only_query';
import { EuiIcon } from '@elastic/eui';
import { i18n } from '@kbn/i18n';
import { DataRequestAbortError } from './util/data_request';
-
-const VISIBILITY_FILTER_CLAUSE = ['all',
- [
- '==',
- ['get', FEATURE_VISIBLE_PROPERTY_NAME],
- true
- ]
-];
-
-const FILL_LAYER_MB_FILTER = [
- ...VISIBILITY_FILTER_CLAUSE,
- [
- 'any',
- ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON],
- ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON]
- ]
-];
-
-const LINE_LAYER_MB_FILTER = [...VISIBILITY_FILTER_CLAUSE,
- [
- 'any',
- ['==', ['geometry-type'], GEO_JSON_TYPE.POLYGON],
- ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POLYGON],
- ['==', ['geometry-type'], GEO_JSON_TYPE.LINE_STRING],
- ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_LINE_STRING]
- ]
-];
-
-const POINT_LAYER_MB_FILTER = [...VISIBILITY_FILTER_CLAUSE,
- [
- 'any',
- ['==', ['geometry-type'], GEO_JSON_TYPE.POINT],
- ['==', ['geometry-type'], GEO_JSON_TYPE.MULTI_POINT]
- ]
-];
-
-
-let idCounter = 0;
-
-function generateNumericalId() {
- const newId = idCounter < Number.MAX_SAFE_INTEGER ? idCounter : 0;
- idCounter = newId + 1;
- return newId;
-}
-
+import { assignFeatureIds } from './util/assign_feature_ids';
+import {
+ getFillFilterExpression,
+ getLineFilterExpression,
+ getPointFilterExpression,
+} from './util/mb_filter_expressions';
export class VectorLayer extends AbstractLayer {
@@ -116,6 +76,10 @@ export class VectorLayer extends AbstractLayer {
});
}
+ _hasJoins() {
+ return this.getValidJoins().length > 0;
+ }
+
isDataLoaded() {
const sourceDataRequest = this.getSourceDataRequest();
if (!sourceDataRequest || !sourceDataRequest.hasData()) {
@@ -478,15 +442,15 @@ export class VectorLayer extends AbstractLayer {
try {
startLoading(SOURCE_DATA_ID_ORIGIN, requestToken, searchFilters);
const layerName = await this.getDisplayName();
- const { data: featureCollection, meta } =
+ const { data: sourceFeatureCollection, meta } =
await this._source.getGeoJsonWithMeta(layerName, searchFilters,
registerCancelCallback.bind(null, requestToken)
);
- this._assignIdsToFeatures(featureCollection);
- stopLoading(SOURCE_DATA_ID_ORIGIN, requestToken, featureCollection, meta);
+ const layerFeatureCollection = assignFeatureIds(sourceFeatureCollection);
+ stopLoading(SOURCE_DATA_ID_ORIGIN, requestToken, layerFeatureCollection, meta);
return {
refreshed: true,
- featureCollection: featureCollection
+ featureCollection: layerFeatureCollection
};
} catch (error) {
if (!(error instanceof DataRequestAbortError)) {
@@ -498,44 +462,21 @@ export class VectorLayer extends AbstractLayer {
}
}
- _assignIdsToFeatures(featureCollection) {
-
- //wrt https://github.com/elastic/kibana/issues/39317
- //In constrained resource environments, mapbox-gl may throw a stackoverflow error due to hitting the browser's recursion limit. This crashes Kibana.
- //This error is thrown in mapbox-gl's quicksort implementation, when it is sorting all the features by id.
- //This is a work-around to avoid hitting such a worst-case
- //This was tested as a suitable work-around for mapbox-gl 0.54
- //The core issue itself is likely related to https://github.com/mapbox/mapbox-gl-js/issues/6086
-
- //This only shuffles the id-assignment, _not_ the features in the collection
- //The reason for this is that we do not want to modify the feature-ordering, which is the responsiblity of the VectorSource#.
- const ids = [];
- for (let i = 0; i < featureCollection.features.length; i++) {
- const id = generateNumericalId();
- ids.push(id);
- }
-
- const randomizedIds = _.shuffle(ids);
- for (let i = 0; i < featureCollection.features.length; i++) {
- const id = randomizedIds[i];
- const feature = featureCollection.features[i];
- feature.id = id; // Mapbox feature state id, must be integer
- }
- }
-
async syncData(syncContext) {
if (!this.isVisible() || !this.showAtZoomLevel(syncContext.dataFilters.zoom)) {
return;
}
const sourceResult = await this._syncSource(syncContext);
- if (!sourceResult.featureCollection || !sourceResult.featureCollection.features.length) {
+ if (
+ !sourceResult.featureCollection ||
+ !sourceResult.featureCollection.features.length ||
+ !this._hasJoins()) {
return;
}
const joinStates = await this._syncJoins(syncContext);
await this._performInnerJoins(sourceResult, joinStates, syncContext.updateSourceData);
-
}
_getSourceFeatureCollection() {
@@ -605,7 +546,11 @@ export class VectorLayer extends AbstractLayer {
source: sourceId,
paint: {}
});
- mbMap.setFilter(pointLayerId, POINT_LAYER_MB_FILTER);
+ }
+
+ const filterExpr = getPointFilterExpression(this._hasJoins());
+ if (filterExpr !== mbMap.getFilter(pointLayerId)) {
+ mbMap.setFilter(pointLayerId, filterExpr);
}
this._style.setMBPaintPropertiesForPoints({
@@ -626,7 +571,11 @@ export class VectorLayer extends AbstractLayer {
type: 'symbol',
source: sourceId,
});
- mbMap.setFilter(symbolLayerId, POINT_LAYER_MB_FILTER);
+ }
+
+ const filterExpr = getPointFilterExpression(this._hasJoins());
+ if (filterExpr !== mbMap.getFilter(symbolLayerId)) {
+ mbMap.setFilter(symbolLayerId, filterExpr);
}
this._style.setMBSymbolPropertiesForPoints({
@@ -640,6 +589,7 @@ export class VectorLayer extends AbstractLayer {
const sourceId = this.getId();
const fillLayerId = this._getMbPolygonLayerId();
const lineLayerId = this._getMbLineLayerId();
+ const hasJoins = this._hasJoins();
if (!mbMap.getLayer(fillLayerId)) {
mbMap.addLayer({
id: fillLayerId,
@@ -647,7 +597,6 @@ export class VectorLayer extends AbstractLayer {
source: sourceId,
paint: {}
});
- mbMap.setFilter(fillLayerId, FILL_LAYER_MB_FILTER);
}
if (!mbMap.getLayer(lineLayerId)) {
mbMap.addLayer({
@@ -656,7 +605,6 @@ export class VectorLayer extends AbstractLayer {
source: sourceId,
paint: {}
});
- mbMap.setFilter(lineLayerId, LINE_LAYER_MB_FILTER);
}
this._style.setMBPaintProperties({
alpha: this.getAlpha(),
@@ -666,9 +614,18 @@ export class VectorLayer extends AbstractLayer {
});
this.syncVisibilityWithMb(mbMap, fillLayerId);
+ mbMap.setLayerZoomRange(fillLayerId, this._descriptor.minZoom, this._descriptor.maxZoom);
+ const fillFilterExpr = getFillFilterExpression(hasJoins);
+ if (fillFilterExpr !== mbMap.getFilter(fillLayerId)) {
+ mbMap.setFilter(fillLayerId, fillFilterExpr);
+ }
+
this.syncVisibilityWithMb(mbMap, lineLayerId);
mbMap.setLayerZoomRange(lineLayerId, this._descriptor.minZoom, this._descriptor.maxZoom);
- mbMap.setLayerZoomRange(fillLayerId, this._descriptor.minZoom, this._descriptor.maxZoom);
+ const lineFilterExpr = getLineFilterExpression(hasJoins);
+ if (lineFilterExpr !== mbMap.getFilter(lineLayerId)) {
+ mbMap.setFilter(lineLayerId, lineFilterExpr);
+ }
}
_syncStylePropertiesWithMb(mbMap) {
diff --git a/x-pack/legacy/plugins/ml/public/application/contexts/kibana/__mocks__/saved_search.ts b/x-pack/legacy/plugins/ml/public/application/contexts/kibana/__mocks__/saved_search.ts
index 2e442c5c61b1e49..2bff760ed3711dd 100644
--- a/x-pack/legacy/plugins/ml/public/application/contexts/kibana/__mocks__/saved_search.ts
+++ b/x-pack/legacy/plugins/ml/public/application/contexts/kibana/__mocks__/saved_search.ts
@@ -5,11 +5,12 @@
*/
import { searchSourceMock } from '../../../../../../../../../src/legacy/ui/public/courier/search_source/mocks';
+import { SearchSourceContract } from '../../../../../../../../../src/legacy/ui/public/courier';
export const savedSearchMock = {
id: 'the-saved-search-id',
title: 'the-saved-search-title',
- searchSource: searchSourceMock,
+ searchSource: searchSourceMock as SearchSourceContract,
columns: [],
sort: [],
destroy: () => {},
diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/index_based/page.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/index_based/page.tsx
index 1caa06862061805..642b4c5649a13d5 100644
--- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/index_based/page.tsx
+++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/index_based/page.tsx
@@ -176,7 +176,7 @@ export const Page: FC = () => {
const searchSource = currentSavedSearch.searchSource;
const query = searchSource.getField('query');
if (query !== undefined) {
- const queryLanguage = query.language;
+ const queryLanguage = query.language as SEARCH_QUERY_LANGUAGE;
const qryString = query.query;
let qry;
if (queryLanguage === SEARCH_QUERY_LANGUAGE.KUERY) {
diff --git a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts
index 0e88b291e76fc95..455fac9b532d61e 100644
--- a/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts
+++ b/x-pack/legacy/plugins/ml/public/application/jobs/new_job/utils/new_job_utils.ts
@@ -7,7 +7,7 @@
import { IndexPattern } from 'ui/index_patterns';
import { SavedSearch } from 'src/legacy/core_plugins/kibana/public/discover/types';
import { KibanaConfigTypeFix } from '../../../contexts/kibana';
-import { esQuery, IIndexPattern } from '../../../../../../../../../src/plugins/data/public';
+import { esQuery, Query, IIndexPattern } from '../../../../../../../../../src/plugins/data/public';
export interface SearchItems {
indexPattern: IIndexPattern;
@@ -28,7 +28,7 @@ export function createSearchItems(
// a lucene query_string.
// Using a blank query will cause match_all:{} to be used
// when passed through luceneStringToDsl
- let query = {
+ let query: Query = {
query: '',
language: 'lucene',
};
@@ -45,12 +45,12 @@ export function createSearchItems(
if (indexPattern.id === undefined && savedSearch.id !== undefined) {
const searchSource = savedSearch.searchSource;
- indexPattern = searchSource.getField('index');
+ indexPattern = searchSource.getField('index')!;
- query = searchSource.getField('query');
+ query = searchSource.getField('query')!;
const fs = searchSource.getField('filter');
- const filters = fs.length ? fs : [];
+ const filters = Array.isArray(fs) ? fs : [];
const esQueryConfigs = esQuery.getEsQueryConfig(kibanaConfig);
combinedQuery = esQuery.buildEsQuery(indexPattern, [query], filters, esQueryConfigs);
diff --git a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts
index a614be547abde86..aeec71462308e2d 100644
--- a/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts
+++ b/x-pack/legacy/plugins/ml/public/application/services/new_job_capabilities_service.ts
@@ -38,7 +38,7 @@ export function loadNewJobCapabilities(
// saved search is being used
// load the index pattern from the saved search
const savedSearch = await savedSearches.get(savedSearchId);
- const indexPattern = savedSearch.searchSource.getField('index');
+ const indexPattern = savedSearch.searchSource.getField('index')!;
await newJobCapsService.initializeFromIndexPattern(indexPattern);
resolve(newJobCapsService.newJobCaps);
} else {
diff --git a/x-pack/legacy/plugins/reporting/common/get_absolute_url.test.ts b/x-pack/legacy/plugins/reporting/common/get_absolute_url.test.ts
index 9bad3b2b08002a6..cb792fbd6ae0397 100644
--- a/x-pack/legacy/plugins/reporting/common/get_absolute_url.test.ts
+++ b/x-pack/legacy/plugins/reporting/common/get_absolute_url.test.ts
@@ -4,88 +4,80 @@
* you may not use this file except in compliance with the Elastic License.
*/
-import { createMockServer } from '../test_helpers/create_mock_server';
import { getAbsoluteUrlFactory } from './get_absolute_url';
-test(`by default it builds url using information from server.info.protocol and the server.config`, () => {
- const mockServer = createMockServer('');
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
+const defaultOptions = {
+ defaultBasePath: 'sbp',
+ protocol: 'http:',
+ hostname: 'localhost',
+ port: 5601,
+};
+
+test(`by default it builds urls using information from server.info.protocol and the server.config`, () => {
+ const getAbsoluteUrl = getAbsoluteUrlFactory(defaultOptions);
const absoluteUrl = getAbsoluteUrl();
expect(absoluteUrl).toBe(`http://localhost:5601/sbp/app/kibana`);
});
test(`uses kibanaServer.protocol if specified`, () => {
- const settings = {
- 'xpack.reporting.kibanaServer.protocol': 'https',
- };
- const mockServer = createMockServer({ settings });
+ const getAbsoluteUrl = getAbsoluteUrlFactory({
+ ...defaultOptions,
+ protocol: 'https:',
+ });
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
const absoluteUrl = getAbsoluteUrl();
expect(absoluteUrl).toBe(`https://localhost:5601/sbp/app/kibana`);
});
test(`uses kibanaServer.hostname if specified`, () => {
- const settings = {
- 'xpack.reporting.kibanaServer.hostname': 'something-else',
- };
- const mockServer = createMockServer({ settings });
+ const getAbsoluteUrl = getAbsoluteUrlFactory({
+ ...defaultOptions,
+ hostname: 'something-else',
+ });
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
const absoluteUrl = getAbsoluteUrl();
expect(absoluteUrl).toBe(`http://something-else:5601/sbp/app/kibana`);
});
test(`uses kibanaServer.port if specified`, () => {
- const settings = {
- 'xpack.reporting.kibanaServer.port': 8008,
- };
- const mockServer = createMockServer({ settings });
+ const getAbsoluteUrl = getAbsoluteUrlFactory({
+ ...defaultOptions,
+ port: 8008,
+ });
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
const absoluteUrl = getAbsoluteUrl();
expect(absoluteUrl).toBe(`http://localhost:8008/sbp/app/kibana`);
});
test(`uses the provided hash`, () => {
- const mockServer = createMockServer('');
-
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
+ const getAbsoluteUrl = getAbsoluteUrlFactory(defaultOptions);
const hash = '/hash';
const absoluteUrl = getAbsoluteUrl({ hash });
expect(absoluteUrl).toBe(`http://localhost:5601/sbp/app/kibana#${hash}`);
});
test(`uses the provided hash with queryString`, () => {
- const mockServer = createMockServer('');
-
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
+ const getAbsoluteUrl = getAbsoluteUrlFactory(defaultOptions);
const hash = '/hash?querystring';
const absoluteUrl = getAbsoluteUrl({ hash });
expect(absoluteUrl).toBe(`http://localhost:5601/sbp/app/kibana#${hash}`);
});
test(`uses the provided basePath`, () => {
- const mockServer = createMockServer('');
-
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
+ const getAbsoluteUrl = getAbsoluteUrlFactory(defaultOptions);
const absoluteUrl = getAbsoluteUrl({ basePath: '/s/marketing' });
expect(absoluteUrl).toBe(`http://localhost:5601/s/marketing/app/kibana`);
});
test(`uses the path`, () => {
- const mockServer = createMockServer('');
-
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
+ const getAbsoluteUrl = getAbsoluteUrlFactory(defaultOptions);
const path = '/app/canvas';
const absoluteUrl = getAbsoluteUrl({ path });
expect(absoluteUrl).toBe(`http://localhost:5601/sbp${path}`);
});
test(`uses the search`, () => {
- const mockServer = createMockServer('');
-
- const getAbsoluteUrl = getAbsoluteUrlFactory(mockServer);
+ const getAbsoluteUrl = getAbsoluteUrlFactory(defaultOptions);
const search = '_t=123456789';
const absoluteUrl = getAbsoluteUrl({ search });
expect(absoluteUrl).toBe(`http://localhost:5601/sbp/app/kibana?${search}`);
diff --git a/x-pack/legacy/plugins/reporting/common/get_absolute_url.ts b/x-pack/legacy/plugins/reporting/common/get_absolute_url.ts
index 1d34189abcb2431..0e350cb1ec01179 100644
--- a/x-pack/legacy/plugins/reporting/common/get_absolute_url.ts
+++ b/x-pack/legacy/plugins/reporting/common/get_absolute_url.ts
@@ -5,24 +5,27 @@
*/
import url from 'url';
-import { ServerFacade } from '../types';
-
-export function getAbsoluteUrlFactory(server: ServerFacade) {
- const config = server.config();
+import { AbsoluteURLFactoryOptions } from '../types';
+export const getAbsoluteUrlFactory = ({
+ protocol,
+ hostname,
+ port,
+ defaultBasePath,
+}: AbsoluteURLFactoryOptions) => {
return function getAbsoluteUrl({
- basePath = config.get('server.basePath'),
+ basePath = defaultBasePath,
hash = '',
path = '/app/kibana',
search = '',
} = {}) {
return url.format({
- protocol: config.get('xpack.reporting.kibanaServer.protocol') || server.info.protocol,
- hostname: config.get('xpack.reporting.kibanaServer.hostname') || config.get('server.host'),
- port: config.get('xpack.reporting.kibanaServer.port') || config.get('server.port'),
+ protocol,
+ hostname,
+ port,
pathname: basePath + path,
hash,
search,
});
};
-}
+};
diff --git a/x-pack/legacy/plugins/reporting/export_types/common/execute_job/get_full_urls.ts b/x-pack/legacy/plugins/reporting/export_types/common/execute_job/get_full_urls.ts
index 3b8285207342173..2b66c77067ed2e6 100644
--- a/x-pack/legacy/plugins/reporting/export_types/common/execute_job/get_full_urls.ts
+++ b/x-pack/legacy/plugins/reporting/export_types/common/execute_job/get_full_urls.ts
@@ -28,7 +28,14 @@ export async function getFullUrls({
job: JobDocPayloadPNG | JobDocPayloadPDF;
server: ServerFacade;
}) {
- const getAbsoluteUrl = getAbsoluteUrlFactory(server);
+ const config = server.config();
+
+ const getAbsoluteUrl = getAbsoluteUrlFactory({
+ defaultBasePath: config.get('server.basePath'),
+ protocol: config.get('xpack.reporting.kibanaServer.protocol') || server.info.protocol,
+ hostname: config.get('xpack.reporting.kibanaServer.hostname') || config.get('server.host'),
+ port: config.get('xpack.reporting.kibanaServer.port') || config.get('server.port'),
+ });
// PDF and PNG job params put in the url differently
let relativeUrls: string[] = [];
diff --git a/x-pack/legacy/plugins/reporting/server/routes/lib/authorized_user_pre_routing.js b/x-pack/legacy/plugins/reporting/server/routes/lib/authorized_user_pre_routing.js
index 10ff9f477f4248b..59317ac46773baa 100644
--- a/x-pack/legacy/plugins/reporting/server/routes/lib/authorized_user_pre_routing.js
+++ b/x-pack/legacy/plugins/reporting/server/routes/lib/authorized_user_pre_routing.js
@@ -6,11 +6,10 @@
import boom from 'boom';
import { getUserFactory } from '../../lib/get_user';
-import { oncePerServer } from '../../lib/once_per_server';
const superuserRole = 'superuser';
-function authorizedUserPreRoutingFn(server) {
+export const authorizedUserPreRoutingFactory = function authorizedUserPreRoutingFn(server) {
const getUser = getUserFactory(server);
const config = server.config();
@@ -40,6 +39,4 @@ function authorizedUserPreRoutingFn(server) {
return user;
};
-}
-
-export const authorizedUserPreRoutingFactory = oncePerServer(authorizedUserPreRoutingFn);
+};
diff --git a/x-pack/legacy/plugins/reporting/server/routes/lib/reporting_feature_pre_routing.js b/x-pack/legacy/plugins/reporting/server/routes/lib/reporting_feature_pre_routing.js
index ad91e5a654a4e4e..92973e3d0b42206 100644
--- a/x-pack/legacy/plugins/reporting/server/routes/lib/reporting_feature_pre_routing.js
+++ b/x-pack/legacy/plugins/reporting/server/routes/lib/reporting_feature_pre_routing.js
@@ -5,9 +5,8 @@
*/
import Boom from 'boom';
-import { oncePerServer } from '../../lib/once_per_server';
-function reportingFeaturePreRoutingFn(server) {
+export const reportingFeaturePreRoutingFactory = function reportingFeaturePreRoutingFn(server) {
const xpackMainPlugin = server.plugins.xpack_main;
const pluginId = 'reporting';
@@ -24,6 +23,4 @@ function reportingFeaturePreRoutingFn(server) {
}
};
};
-}
-
-export const reportingFeaturePreRoutingFactory = oncePerServer(reportingFeaturePreRoutingFn);
+};
diff --git a/x-pack/legacy/plugins/reporting/types.d.ts b/x-pack/legacy/plugins/reporting/types.d.ts
index 7d05811ef4aa644..e8fb015426f5156 100644
--- a/x-pack/legacy/plugins/reporting/types.d.ts
+++ b/x-pack/legacy/plugins/reporting/types.d.ts
@@ -326,3 +326,10 @@ export { CancellationToken } from './common/cancellation_token';
// Prefer to import this type using: `import { LevelLogger } from 'relative/path/server/lib';`
export { LevelLogger as Logger } from './server/lib/level_logger';
+
+export interface AbsoluteURLFactoryOptions {
+ defaultBasePath: string;
+ protocol: string;
+ hostname: string;
+ port: string | number;
+}
diff --git a/x-pack/legacy/plugins/rollup/public/search/register.js b/x-pack/legacy/plugins/rollup/public/search/register.js
index 917ee872254f561..f7f1c681b63caac 100644
--- a/x-pack/legacy/plugins/rollup/public/search/register.js
+++ b/x-pack/legacy/plugins/rollup/public/search/register.js
@@ -4,7 +4,7 @@
* you may not use this file except in compliance with the Elastic License.
*/
-import { addSearchStrategy } from 'ui/courier';
+import { addSearchStrategy } from '../../../../../../src/legacy/ui/public/courier';
import { rollupSearchStrategy } from './rollup_search_strategy';
export function initSearch() {
diff --git a/x-pack/legacy/plugins/rollup/public/search/rollup_search_strategy.js b/x-pack/legacy/plugins/rollup/public/search/rollup_search_strategy.js
index ab24a37a2ecec47..28f08ba1ab952f8 100644
--- a/x-pack/legacy/plugins/rollup/public/search/rollup_search_strategy.js
+++ b/x-pack/legacy/plugins/rollup/public/search/rollup_search_strategy.js
@@ -5,7 +5,7 @@
*/
import { kfetch } from 'ui/kfetch';
-import { SearchError, getSearchErrorType } from 'ui/courier';
+import { SearchError, getSearchErrorType } from '../../../../../../src/legacy/ui/public/courier';
function serializeFetchParams(searchRequests) {
return JSON.stringify(searchRequests.map(searchRequestWithFetchParams => {
diff --git a/x-pack/legacy/plugins/security/public/views/management/edit_user/components/edit_user_page.tsx b/x-pack/legacy/plugins/security/public/views/management/edit_user/components/edit_user_page.tsx
index 282ce4eea160cb4..91f5f048adc6d04 100644
--- a/x-pack/legacy/plugins/security/public/views/management/edit_user/components/edit_user_page.tsx
+++ b/x-pack/legacy/plugins/security/public/views/management/edit_user/components/edit_user_page.tsx
@@ -374,7 +374,7 @@ class EditUserPageUI extends Component
{
-
+
{isNewUser ? (
{
values={{ userName: user.username }}
/>
)}
-
+
{reserved && (
diff --git a/x-pack/legacy/plugins/siem/common/constants.ts b/x-pack/legacy/plugins/siem/common/constants.ts
index f01493cec869e52..e5d1fc83dac2645 100644
--- a/x-pack/legacy/plugins/siem/common/constants.ts
+++ b/x-pack/legacy/plugins/siem/common/constants.ts
@@ -17,6 +17,8 @@ export const DEFAULT_SIEM_TIME_RANGE = 'siem:timeDefaults';
export const DEFAULT_SIEM_REFRESH_INTERVAL = 'siem:refreshIntervalDefaults';
export const DEFAULT_SIGNALS_INDEX_KEY = 'siem:defaultSignalsIndex';
export const DEFAULT_SIGNALS_INDEX = '.siem-signals';
+export const DEFAULT_MAX_SIGNALS = 100;
+export const DEFAULT_SEARCH_AFTER_PAGE_SIZE = 100;
export const DEFAULT_ANOMALY_SCORE = 'siem:defaultAnomalyScore';
export const DEFAULT_MAX_TABLE_QUERY_SIZE = 10000;
export const DEFAULT_SCALE_DATE_FORMAT = 'dateFormat:scaled';
diff --git a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js
index 597d93a44210bc5..263a2a59de31f58 100644
--- a/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js
+++ b/x-pack/legacy/plugins/siem/scripts/convert_saved_search_to_signals.js
@@ -34,9 +34,17 @@ const TYPE = 'query';
const FROM = 'now-6m';
const TO = 'now';
const IMMUTABLE = true;
-const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'];
-const OUTPUT_INDEX = process.env.SIGNALS_INDEX || '.siem-signals';
const RISK_SCORE = 50;
+const ENABLED = false;
+let allSignals = '';
+const allSignalsNdJson = 'all_rules.ndjson';
+
+// For converting, if you want to use these instead of rely on the defaults then
+// comment these in and use them for the script. Otherwise this is commented out
+// so we can utilize the defaults of input and output which are based on saved objects
+// of siem:defaultIndex and siem:defaultSignalsIndex
+// const INDEX = ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'];
+// const OUTPUT_INDEX = process.env.SIGNALS_INDEX || '.siem-signals';
const walk = dir => {
const list = fs.readdirSync(dir);
@@ -124,7 +132,6 @@ async function main() {
risk_score: RISK_SCORE,
description: description || title,
immutable: IMMUTABLE,
- index: INDEX,
interval: INTERVAL,
name: title,
severity: SEVERITY,
@@ -134,16 +141,22 @@ async function main() {
query,
language,
filters: filter,
- output_index: OUTPUT_INDEX,
+ enabled: ENABLED,
+ // comment these in if you want to use these for input output, otherwise
+ // with these two commented out, we will use the default saved objects from spaces.
+ // index: INDEX,
+ // output_index: OUTPUT_INDEX,
};
fs.writeFileSync(
`${outputDir}/${fileToWrite}.json`,
JSON.stringify(outputMessage, null, 2)
);
+ allSignals += `${JSON.stringify(outputMessage)}\n`;
}
}
);
+ fs.writeFileSync(`${outputDir}/${allSignalsNdJson}`, allSignals);
}
if (require.main === module) {
diff --git a/x-pack/legacy/plugins/siem/server/kibana.index.ts b/x-pack/legacy/plugins/siem/server/kibana.index.ts
index c79b2651c11cb1b..a92bca064dab9c1 100644
--- a/x-pack/legacy/plugins/siem/server/kibana.index.ts
+++ b/x-pack/legacy/plugins/siem/server/kibana.index.ts
@@ -32,7 +32,8 @@ export const initServerWithKibana = (
mode: EnvironmentMode
) => {
if (kbnServer.plugins.alerting != null) {
- const type = signalsAlertType({ logger });
+ const version = kbnServer.config().get('pkg.version');
+ const type = signalsAlertType({ logger, version });
if (isAlertExecutor(type)) {
kbnServer.plugins.alerting.setup.registerType(type);
}
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md
index 0a0439a9ace1b05..5d9d87a1cbc2fda 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/README.md
@@ -104,6 +104,13 @@ You should also see the SIEM detect the feature flags and start the API endpoint
server log [11:39:05.561] [info][siem] Detected feature flags for actions and alerting and enabling signals API endpoints
```
+Go into your SIEM Advanced settings and underneath the setting of `siem:defaultSignalsIndex`, set that to the same
+value as you did with the environment variable of SIGNALS_INDEX, which should be `.siem-signals-${your user id}`
+
+```
+.siem-signals-${your user id}
+```
+
Open a terminal and go into the scripts folder `cd kibana/x-pack/legacy/plugins/siem/server/lib/detection_engine/scripts` and run:
```sh
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts
index 0a70a7342b2dd3c..7d3b51c071c091e 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/__mocks__/es_results.ts
@@ -30,33 +30,43 @@ export const sampleSignalAlertParams = (
filters: undefined,
savedId: undefined,
meta: undefined,
- size: 1000,
});
-export const sampleDocNoSortId: SignalSourceHit = {
+export const sampleDocNoSortId = (someUuid: string): SignalSourceHit => ({
_index: 'myFakeSignalIndex',
_type: 'doc',
_score: 100,
_version: 1,
- _id: 'someFakeId',
+ _id: someUuid,
_source: {
someKey: 'someValue',
'@timestamp': 'someTimeStamp',
},
-};
+});
+
+export const sampleDocNoSortIdNoVersion = (someUuid: string): SignalSourceHit => ({
+ _index: 'myFakeSignalIndex',
+ _type: 'doc',
+ _score: 100,
+ _id: someUuid,
+ _source: {
+ someKey: 'someValue',
+ '@timestamp': 'someTimeStamp',
+ },
+});
-export const sampleDocWithSortId: SignalSourceHit = {
+export const sampleDocWithSortId = (someUuid: string): SignalSourceHit => ({
_index: 'myFakeSignalIndex',
_type: 'doc',
_score: 100,
_version: 1,
- _id: 'someFakeId',
+ _id: someUuid,
_source: {
someKey: 'someValue',
'@timestamp': 'someTimeStamp',
},
sort: ['1234567891111'],
-};
+});
export const sampleEmptyDocSearchResults: SignalSearchResponse = {
took: 10,
@@ -74,7 +84,61 @@ export const sampleEmptyDocSearchResults: SignalSearchResponse = {
},
};
-export const sampleDocSearchResultsNoSortId: SignalSearchResponse = {
+export const sampleBulkCreateDuplicateResult = {
+ took: 60,
+ errors: true,
+ items: [
+ {
+ create: {
+ _index: 'test',
+ _type: '_doc',
+ _id: '4',
+ _version: 1,
+ result: 'created',
+ _shards: {
+ total: 2,
+ successful: 1,
+ failed: 0,
+ },
+ _seq_no: 1,
+ _primary_term: 1,
+ status: 201,
+ },
+ },
+ {
+ create: {
+ _index: 'test',
+ _type: '_doc',
+ _id: '4',
+ status: 409,
+ error: {
+ type: 'version_conflict_engine_exception',
+ reason: '[4]: version conflict, document already exists (current version [1])',
+ index_uuid: 'cXmq4Rt3RGGswDTTwZFzvA',
+ shard: '0',
+ index: 'test',
+ },
+ },
+ },
+ {
+ create: {
+ _index: 'test',
+ _type: '_doc',
+ _id: '4',
+ status: 409,
+ error: {
+ type: 'version_conflict_engine_exception',
+ reason: '[4]: version conflict, document already exists (current version [1])',
+ index_uuid: 'cXmq4Rt3RGGswDTTwZFzvA',
+ shard: '0',
+ index: 'test',
+ },
+ },
+ },
+ ],
+};
+
+export const sampleDocSearchResultsNoSortId = (someUuid: string): SignalSearchResponse => ({
took: 10,
timed_out: false,
_shards: {
@@ -88,13 +152,35 @@ export const sampleDocSearchResultsNoSortId: SignalSearchResponse = {
max_score: 100,
hits: [
{
- ...sampleDocNoSortId,
+ ...sampleDocNoSortId(someUuid),
},
],
},
-};
+});
+
+export const sampleDocSearchResultsNoSortIdNoVersion = (
+ someUuid: string
+): SignalSearchResponse => ({
+ took: 10,
+ timed_out: false,
+ _shards: {
+ total: 10,
+ successful: 10,
+ failed: 0,
+ skipped: 0,
+ },
+ hits: {
+ total: 100,
+ max_score: 100,
+ hits: [
+ {
+ ...sampleDocNoSortIdNoVersion(someUuid),
+ },
+ ],
+ },
+});
-export const sampleDocSearchResultsNoSortIdNoHits: SignalSearchResponse = {
+export const sampleDocSearchResultsNoSortIdNoHits = (someUuid: string): SignalSearchResponse => ({
took: 10,
timed_out: false,
_shards: {
@@ -108,13 +194,17 @@ export const sampleDocSearchResultsNoSortIdNoHits: SignalSearchResponse = {
max_score: 100,
hits: [
{
- ...sampleDocNoSortId,
+ ...sampleDocNoSortId(someUuid),
},
],
},
-};
+});
-export const repeatedSearchResultsWithSortId = (repeat: number) => ({
+export const repeatedSearchResultsWithSortId = (
+ total: number,
+ pageSize: number,
+ guids: string[]
+) => ({
took: 10,
timed_out: false,
_shards: {
@@ -124,15 +214,15 @@ export const repeatedSearchResultsWithSortId = (repeat: number) => ({
skipped: 0,
},
hits: {
- total: repeat,
+ total,
max_score: 100,
- hits: Array.from({ length: repeat }).map(x => ({
- ...sampleDocWithSortId,
+ hits: Array.from({ length: pageSize }).map((x, index) => ({
+ ...sampleDocWithSortId(guids[index]),
})),
},
});
-export const sampleDocSearchResultsWithSortId: SignalSearchResponse = {
+export const sampleDocSearchResultsWithSortId = (someUuid: string): SignalSearchResponse => ({
took: 10,
timed_out: false,
_shards: {
@@ -146,10 +236,10 @@ export const sampleDocSearchResultsWithSortId: SignalSearchResponse = {
max_score: 100,
hits: [
{
- ...sampleDocWithSortId,
+ ...sampleDocWithSortId(someUuid),
},
],
},
-};
+});
export const sampleSignalId = '04128c15-0d1b-4716-a4c5-46997ac7f3bd';
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts
index 420f99543142320..8770282356cf561 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/create_signals.ts
@@ -29,7 +29,6 @@ export const createSignals = async ({
outputIndex,
name,
severity,
- size,
tags,
to,
type,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.test.ts
new file mode 100644
index 000000000000000..07eb7c885b4435d
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.test.ts
@@ -0,0 +1,286 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { savedObjectsClientMock } from 'src/core/server/mocks';
+import {
+ DEFAULT_SIGNALS_INDEX_KEY,
+ DEFAULT_INDEX_KEY,
+ DEFAULT_SIGNALS_INDEX,
+} from '../../../../common/constants';
+import { AlertServices } from '../../../../../alerting/server/types';
+import { getInputOutputIndex, getOutputIndex, getInputIndex } from './get_input_output_index';
+import { defaultIndexPattern } from '../../../../default_index_pattern';
+
+describe('get_input_output_index', () => {
+ let savedObjectsClient = savedObjectsClientMock.create();
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ let servicesMock: AlertServices = {
+ savedObjectsClient,
+ callCluster: jest.fn(),
+ alertInstanceFactory: jest.fn(),
+ };
+
+ beforeAll(() => {
+ jest.resetAllMocks();
+ });
+
+ afterAll(() => {
+ jest.resetAllMocks();
+ });
+
+ beforeEach(() => {
+ savedObjectsClient = savedObjectsClientMock.create();
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ servicesMock = {
+ savedObjectsClient,
+ callCluster: jest.fn(),
+ alertInstanceFactory: jest.fn(),
+ };
+ });
+
+ describe('getInputOutputIndex', () => {
+ test('Returns inputIndex as is if inputIndex and outputIndex are both passed in', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ const { inputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ ['test-input-index-1'],
+ 'test-output-index'
+ );
+ expect(inputIndex).toEqual(['test-input-index-1']);
+ });
+
+ test('Returns outputIndex as is if inputIndex and outputIndex are both passed in', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ const { outputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ ['test-input-index-1'],
+ 'test-output-index'
+ );
+ expect(outputIndex).toEqual('test-output-index');
+ });
+
+ test('Returns inputIndex as is if inputIndex is defined but outputIndex is null', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ const { inputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ ['test-input-index-1'],
+ null
+ );
+ expect(inputIndex).toEqual(['test-input-index-1']);
+ });
+
+ test('Returns outputIndex as is if inputIndex is null but outputIndex is defined', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ const { outputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ null,
+ 'test-output-index'
+ );
+ expect(outputIndex).toEqual('test-output-index');
+ });
+
+ test('Returns a saved object outputIndex if both passed in are undefined', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_SIGNALS_INDEX_KEY]: '.signals-test-index',
+ },
+ }));
+ const { outputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ undefined,
+ undefined
+ );
+ expect(outputIndex).toEqual('.signals-test-index');
+ });
+
+ test('Returns a saved object outputIndex if passed in outputIndex is undefined', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_SIGNALS_INDEX_KEY]: '.signals-test-index',
+ },
+ }));
+ const { outputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ ['test-input-index-1'],
+ undefined
+ );
+ expect(outputIndex).toEqual('.signals-test-index');
+ });
+
+ test('Returns a saved object outputIndex default from constants if both passed in input and configuration are null', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_SIGNALS_INDEX_KEY]: null,
+ },
+ }));
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ const { outputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', null, null);
+ expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX);
+ });
+
+ test('Returns a saved object outputIndex default from constants if both passed in input and configuration are missing', async () => {
+ const { outputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ undefined,
+ undefined
+ );
+ expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX);
+ });
+
+ test('Returns a saved object inputIndex if passed in inputIndex and outputIndex are undefined', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_INDEX_KEY]: ['configured-index-1', 'configured-index-2'],
+ },
+ }));
+ const { inputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', undefined, undefined);
+ expect(inputIndex).toEqual(['configured-index-1', 'configured-index-2']);
+ });
+
+ test('Returns a saved object inputIndex if passed in inputIndex is undefined', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_INDEX_KEY]: ['configured-index-1', 'configured-index-2'],
+ },
+ }));
+ const { inputIndex } = await getInputOutputIndex(
+ servicesMock,
+ '8.0.0',
+ undefined,
+ 'output-index-1'
+ );
+ expect(inputIndex).toEqual(['configured-index-1', 'configured-index-2']);
+ });
+
+ test('Returns a saved object inputIndex default from constants if both passed in inputIndex and configuration is null', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_INDEX_KEY]: null,
+ },
+ }));
+ const { inputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', null, null);
+ expect(inputIndex).toEqual(defaultIndexPattern);
+ });
+
+ test('Returns a saved object inputIndex default from constants if both passed in inputIndex and configuration attributes is missing', async () => {
+ const { inputIndex } = await getInputOutputIndex(servicesMock, '8.0.0', undefined, undefined);
+ expect(inputIndex).toEqual(defaultIndexPattern);
+ });
+ });
+
+ describe('getOutputIndex', () => {
+ test('test output index is returned when passed in as is', async () => {
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const outputIndex = getOutputIndex('output-index-1', mockConfiguration);
+ expect(outputIndex).toEqual('output-index-1');
+ });
+
+ test('configured output index is returned when output index is null', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_SIGNALS_INDEX_KEY]: '.siem-test-signals',
+ },
+ }));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const outputIndex = getOutputIndex(null, mockConfiguration);
+ expect(outputIndex).toEqual('.siem-test-signals');
+ });
+
+ test('output index from constants is returned when output index is null and so is the configuration', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_SIGNALS_INDEX_KEY]: null,
+ },
+ }));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const outputIndex = getOutputIndex(null, mockConfiguration);
+ expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX);
+ });
+
+ test('output index from constants is returned when output index is null and configuration is missing', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const outputIndex = getOutputIndex(null, mockConfiguration);
+ expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX);
+ });
+
+ test('output index from constants is returned when output index is null and attributes is missing', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({}));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const outputIndex = getOutputIndex(null, mockConfiguration);
+ expect(outputIndex).toEqual(DEFAULT_SIGNALS_INDEX);
+ });
+ });
+
+ describe('getInputIndex', () => {
+ test('test input index is returned when passed in as is', async () => {
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const inputIndex = getInputIndex(['input-index-1'], mockConfiguration);
+ expect(inputIndex).toEqual(['input-index-1']);
+ });
+
+ test('configured input index is returned when input index is null', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_INDEX_KEY]: ['input-index-1', 'input-index-2'],
+ },
+ }));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const inputIndex = getInputIndex(null, mockConfiguration);
+ expect(inputIndex).toEqual(['input-index-1', 'input-index-2']);
+ });
+
+ test('input index from constants is returned when input index is null and so is the configuration', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {
+ [DEFAULT_INDEX_KEY]: null,
+ },
+ }));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const inputIndex = getInputIndex(null, mockConfiguration);
+ expect(inputIndex).toEqual(defaultIndexPattern);
+ });
+
+ test('input index from constants is returned when input index is null and configuration is missing', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({
+ attributes: {},
+ }));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const inputIndex = getInputIndex(null, mockConfiguration);
+ expect(inputIndex).toEqual(defaultIndexPattern);
+ });
+
+ test('input index from constants is returned when input index is null and attributes is missing', async () => {
+ savedObjectsClient.get = jest.fn().mockImplementation(() => ({}));
+ const mockConfiguration = await savedObjectsClient.get('config', '8.0.0');
+ const inputIndex = getInputIndex(null, mockConfiguration);
+ expect(inputIndex).toEqual(defaultIndexPattern);
+ });
+ });
+});
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.ts
new file mode 100644
index 000000000000000..567ab27976d8d20
--- /dev/null
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/get_input_output_index.ts
@@ -0,0 +1,72 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { SavedObject, SavedObjectAttributes } from 'src/core/server';
+import { defaultIndexPattern } from '../../../../default_index_pattern';
+import { AlertServices } from '../../../../../alerting/server/types';
+import {
+ DEFAULT_INDEX_KEY,
+ DEFAULT_SIGNALS_INDEX_KEY,
+ DEFAULT_SIGNALS_INDEX,
+} from '../../../../common/constants';
+
+interface IndexObjectAttributes extends SavedObjectAttributes {
+ [DEFAULT_INDEX_KEY]: string[];
+ [DEFAULT_SIGNALS_INDEX_KEY]: string;
+}
+
+export const getInputIndex = (
+ inputIndex: string[] | undefined | null,
+ configuration: SavedObject
+): string[] => {
+ if (inputIndex != null) {
+ return inputIndex;
+ } else {
+ if (configuration.attributes != null && configuration.attributes[DEFAULT_INDEX_KEY] != null) {
+ return configuration.attributes[DEFAULT_INDEX_KEY];
+ } else {
+ return defaultIndexPattern;
+ }
+ }
+};
+
+export const getOutputIndex = (
+ outputIndex: string | undefined | null,
+ configuration: SavedObject
+): string => {
+ if (outputIndex != null) {
+ return outputIndex;
+ } else {
+ if (
+ configuration.attributes != null &&
+ configuration.attributes[DEFAULT_SIGNALS_INDEX_KEY] != null
+ ) {
+ return configuration.attributes[DEFAULT_SIGNALS_INDEX_KEY];
+ } else {
+ return DEFAULT_SIGNALS_INDEX;
+ }
+ }
+};
+
+export const getInputOutputIndex = async (
+ services: AlertServices,
+ version: string,
+ inputIndex: string[] | null | undefined,
+ outputIndex: string | null | undefined
+): Promise<{
+ inputIndex: string[];
+ outputIndex: string;
+}> => {
+ if (inputIndex != null && outputIndex != null) {
+ return { inputIndex, outputIndex };
+ } else {
+ const configuration = await services.savedObjectsClient.get('config', version);
+ return {
+ inputIndex: getInputIndex(inputIndex, configuration),
+ outputIndex: getOutputIndex(outputIndex, configuration),
+ };
+ }
+};
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts
index 8308bca68e9af5d..69eb3eb6650608c 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/signals_alert_type.ts
@@ -6,14 +6,25 @@
import { schema } from '@kbn/config-schema';
import { Logger } from 'src/core/server';
-import { SIGNALS_ID } from '../../../../common/constants';
+import {
+ SIGNALS_ID,
+ DEFAULT_MAX_SIGNALS,
+ DEFAULT_SEARCH_AFTER_PAGE_SIZE,
+} from '../../../../common/constants';
import { buildEventsSearchQuery } from './build_events_query';
-import { searchAfterAndBulkIndex } from './utils';
+import { searchAfterAndBulkCreate } from './utils';
import { SignalAlertTypeDefinition } from './types';
import { getFilter } from './get_filter';
+import { getInputOutputIndex } from './get_input_output_index';
-export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTypeDefinition => {
+export const signalsAlertType = ({
+ logger,
+ version,
+}: {
+ logger: Logger;
+ version: string;
+}): SignalAlertTypeDefinition => {
return {
id: SIGNALS_ID,
name: 'SIEM Signals',
@@ -26,21 +37,20 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
filter: schema.nullable(schema.object({}, { allowUnknowns: true })),
ruleId: schema.string(),
immutable: schema.boolean({ defaultValue: false }),
- index: schema.arrayOf(schema.string()),
+ index: schema.nullable(schema.arrayOf(schema.string())),
language: schema.nullable(schema.string()),
- outputIndex: schema.string(),
+ outputIndex: schema.nullable(schema.string()),
savedId: schema.nullable(schema.string()),
meta: schema.nullable(schema.object({}, { allowUnknowns: true })),
query: schema.nullable(schema.string()),
filters: schema.nullable(schema.arrayOf(schema.object({}, { allowUnknowns: true }))),
- maxSignals: schema.number({ defaultValue: 10000 }),
+ maxSignals: schema.number({ defaultValue: DEFAULT_MAX_SIGNALS }),
riskScore: schema.number(),
severity: schema.string(),
tags: schema.arrayOf(schema.string(), { defaultValue: [] }),
to: schema.string(),
type: schema.string(),
references: schema.arrayOf(schema.string(), { defaultValue: [] }),
- size: schema.maybe(schema.number()),
}),
},
async executor({ alertId, services, params }) {
@@ -56,7 +66,6 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
query,
to,
type,
- size,
} = params;
// TODO: Remove this hard extraction of name once this is fixed: https://github.com/elastic/kibana/issues/50522
@@ -68,8 +77,18 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
const interval: string = savedObject.attributes.interval;
const enabled: boolean = savedObject.attributes.enabled;
- const searchAfterSize = size ? size : 1000;
+ // set searchAfter page size to be the lesser of default page size or maxSignals.
+ const searchAfterSize =
+ DEFAULT_SEARCH_AFTER_PAGE_SIZE <= params.maxSignals
+ ? DEFAULT_SEARCH_AFTER_PAGE_SIZE
+ : params.maxSignals;
+ const { inputIndex, outputIndex: signalsIndex } = await getInputOutputIndex(
+ services,
+ version,
+ index,
+ outputIndex
+ );
const esFilter = await getFilter({
type,
filter,
@@ -78,11 +97,11 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
query,
savedId,
services,
- index,
+ index: inputIndex,
});
const noReIndex = buildEventsSearchQuery({
- index,
+ index: inputIndex,
from,
to,
filter: esFilter,
@@ -98,22 +117,27 @@ export const signalsAlertType = ({ logger }: { logger: Logger }): SignalAlertTyp
const noReIndexResult = await services.callCluster('search', noReIndex);
if (noReIndexResult.hits.total.value !== 0) {
logger.info(
- `Total signals found from signal rule "id: ${alertId}", "ruleId: ${ruleId}": ${noReIndexResult.hits.total.value}`
+ `Found ${
+ noReIndexResult.hits.total.value
+ } signals from the indexes of "${inputIndex.join(
+ ', '
+ )}" using signal rule "id: ${alertId}", "ruleId: ${ruleId}", pushing signals to index ${signalsIndex}`
);
}
- const bulkIndexResult = await searchAfterAndBulkIndex({
+ const bulkIndexResult = await searchAfterAndBulkCreate({
someResult: noReIndexResult,
signalParams: params,
services,
logger,
id: alertId,
- signalsIndex: outputIndex,
+ signalsIndex,
name,
createdBy,
updatedBy,
interval,
enabled,
+ pageSize: searchAfterSize,
});
if (bulkIndexResult) {
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts
index 9c6e1f99c672b25..29eb7872f163dcb 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/types.ts
@@ -42,7 +42,6 @@ export interface SignalAlertParams {
savedId: string | undefined | null;
meta: Record | undefined | null;
severity: string;
- size: number | undefined | null;
tags: string[];
to: string;
type: 'filter' | 'query' | 'saved_query';
@@ -173,7 +172,46 @@ export interface SignalSource {
export interface BulkResponse {
took: number;
errors: boolean;
- items: unknown[];
+ items: [
+ {
+ create: {
+ _index: string;
+ _type?: string;
+ _id: string;
+ _version: number;
+ result?: string;
+ _shards?: {
+ total: number;
+ successful: number;
+ failed: number;
+ };
+ _seq_no?: number;
+ _primary_term?: number;
+ status: number;
+ error?: {
+ type: string;
+ reason: string;
+ index_uuid?: string;
+ shard: string;
+ index: string;
+ };
+ };
+ }
+ ];
+}
+
+export interface MGetResponse {
+ docs: GetResponse[];
+}
+export interface GetResponse {
+ _index: string;
+ _type: string;
+ _id: string;
+ _version: number;
+ _seq_no: number;
+ _primary_term: number;
+ found: boolean;
+ _source: SearchTypes;
}
export type SignalSearchResponse = SearchResponse;
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts
index bc147fa1dae07cd..4aac425c7f80f8e 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.test.ts
@@ -3,23 +3,28 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
+
+import uuid from 'uuid';
import { savedObjectsClientMock } from 'src/core/server/mocks';
import { Logger } from '../../../../../../../../src/core/server';
import {
buildBulkBody,
- singleBulkIndex,
+ generateId,
+ singleBulkCreate,
singleSearchAfter,
- searchAfterAndBulkIndex,
+ searchAfterAndBulkCreate,
} from './utils';
import {
sampleDocNoSortId,
sampleSignalAlertParams,
sampleDocSearchResultsNoSortId,
sampleDocSearchResultsNoSortIdNoHits,
+ sampleDocSearchResultsNoSortIdNoVersion,
sampleDocSearchResultsWithSortId,
sampleEmptyDocSearchResults,
repeatedSearchResultsWithSortId,
+ sampleBulkCreateDuplicateResult,
sampleSignalId,
} from './__mocks__/es_results';
import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants';
@@ -46,9 +51,10 @@ describe('utils', () => {
});
describe('buildBulkBody', () => {
test('if bulk body builds well-defined body', () => {
+ const fakeUuid = uuid.v4();
const sampleParams = sampleSignalAlertParams(undefined);
const fakeSignalSourceHit = buildBulkBody({
- doc: sampleDocNoSortId,
+ doc: sampleDocNoSortId(fakeUuid),
signalParams: sampleParams,
id: sampleSignalId,
name: 'rule-name',
@@ -59,11 +65,13 @@ describe('utils', () => {
});
// Timestamp will potentially always be different so remove it for the test
delete fakeSignalSourceHit['@timestamp'];
+ if (fakeSignalSourceHit.signal.parent) {
+ delete fakeSignalSourceHit.signal.parent.id;
+ }
expect(fakeSignalSourceHit).toEqual({
someKey: 'someValue',
signal: {
parent: {
- id: 'someFakeId',
type: 'event',
index: 'myFakeSignalIndex',
depth: 1,
@@ -88,7 +96,6 @@ describe('utils', () => {
severity: 'high',
tags: ['some fake tag'],
type: 'query',
- size: 1000,
status: 'open',
to: 'now',
enabled: true,
@@ -99,8 +106,114 @@ describe('utils', () => {
});
});
});
- describe('singleBulkIndex', () => {
- test('create successful bulk index', async () => {
+ describe('singleBulkCreate', () => {
+ describe('create signal id gereateId', () => {
+ test('two docs with same index, id, and version should have same id', () => {
+ const findex = 'myfakeindex';
+ const fid = 'somefakeid';
+ const version = '1';
+ const ruleId = 'rule-1';
+ // 'myfakeindexsomefakeid1rule-1'
+ const generatedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679';
+ const firstHash = generateId(findex, fid, version, ruleId);
+ const secondHash = generateId(findex, fid, version, ruleId);
+ expect(firstHash).toEqual(generatedHash);
+ expect(secondHash).toEqual(generatedHash);
+ expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field
+ expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512);
+ });
+ test('two docs with different index, id, and version should have different id', () => {
+ const findex = 'myfakeindex';
+ const findex2 = 'mysecondfakeindex';
+ const fid = 'somefakeid';
+ const version = '1';
+ const ruleId = 'rule-1';
+ // 'myfakeindexsomefakeid1rule-1'
+ const firstGeneratedHash =
+ '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679';
+ // 'mysecondfakeindexsomefakeid1rule-1'
+ const secondGeneratedHash =
+ 'a852941273f805ffe9006e574601acc8ae1148d6c0b3f7f8c4785cba8f6b768a';
+ const firstHash = generateId(findex, fid, version, ruleId);
+ const secondHash = generateId(findex2, fid, version, ruleId);
+ expect(firstHash).toEqual(firstGeneratedHash);
+ expect(secondHash).toEqual(secondGeneratedHash);
+ expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field
+ expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512);
+ expect(firstHash).not.toEqual(secondHash);
+ });
+ test('two docs with same index, different id, and same version should have different id', () => {
+ const findex = 'myfakeindex';
+ const fid = 'somefakeid';
+ const fid2 = 'somefakeid2';
+ const version = '1';
+ const ruleId = 'rule-1';
+ // 'myfakeindexsomefakeid1rule-1'
+ const firstGeneratedHash =
+ '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679';
+ // 'myfakeindexsomefakeid21rule-1'
+ const secondGeneratedHash =
+ '7d33faea18159fd010c4b79890620e8b12cdc88ec1d370149d0e5552ce860255';
+ const firstHash = generateId(findex, fid, version, ruleId);
+ const secondHash = generateId(findex, fid2, version, ruleId);
+ expect(firstHash).toEqual(firstGeneratedHash);
+ expect(secondHash).toEqual(secondGeneratedHash);
+ expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field
+ expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512);
+ expect(firstHash).not.toEqual(secondHash);
+ });
+ test('two docs with same index, same id, and different version should have different id', () => {
+ const findex = 'myfakeindex';
+ const fid = 'somefakeid';
+ const version = '1';
+ const version2 = '2';
+ const ruleId = 'rule-1';
+ // 'myfakeindexsomefakeid1rule-1'
+ const firstGeneratedHash =
+ '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679';
+ // myfakeindexsomefakeid2rule-1'
+ const secondGeneratedHash =
+ 'f016f3071fa9df9221d2fb2ba92389d4d388a4347c6ec7a4012c01cb1c640a40';
+ const firstHash = generateId(findex, fid, version, ruleId);
+ const secondHash = generateId(findex, fid, version2, ruleId);
+ expect(firstHash).toEqual(firstGeneratedHash);
+ expect(secondHash).toEqual(secondGeneratedHash);
+ expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field
+ expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512);
+ expect(firstHash).not.toEqual(secondHash);
+ });
+ test('Ensure generated id is less than 512 bytes, even for really really long strings', () => {
+ const longIndexName =
+ 'myfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindex';
+ const fid = 'somefakeid';
+ const version = '1';
+ const ruleId = 'rule-1';
+ const firstHash = generateId(longIndexName, fid, version, ruleId);
+ expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field
+ });
+ test('two docs with same index, same id, same version number, and different rule ids should have different id', () => {
+ const findex = 'myfakeindex';
+ const fid = 'somefakeid';
+ const version = '1';
+ const ruleId = 'rule-1';
+ const ruleId2 = 'rule-2';
+ // 'myfakeindexsomefakeid1rule-1'
+ const firstGeneratedHash =
+ '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679';
+ // myfakeindexsomefakeid1rule-2'
+ const secondGeneratedHash =
+ '1eb04f997086f8b3b143d4d9b18ac178c4a7423f71a5dad9ba8b9e92603c6863';
+ const firstHash = generateId(findex, fid, version, ruleId);
+ const secondHash = generateId(findex, fid, version, ruleId2);
+ expect(firstHash).toEqual(firstGeneratedHash);
+ expect(secondHash).toEqual(secondGeneratedHash);
+ expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field
+ expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512);
+ expect(firstHash).not.toEqual(secondHash);
+ });
+ });
+ test('create successful bulk create', async () => {
+ const fakeUuid = uuid.v4();
const sampleParams = sampleSignalAlertParams(undefined);
const sampleSearchResult = sampleDocSearchResultsNoSortId;
mockService.callCluster.mockReturnValueOnce({
@@ -112,8 +225,36 @@ describe('utils', () => {
},
],
});
- const successfulSingleBulkIndex = await singleBulkIndex({
- someResult: sampleSearchResult,
+ const successfulsingleBulkCreate = await singleBulkCreate({
+ someResult: sampleSearchResult(fakeUuid),
+ signalParams: sampleParams,
+ services: mockService,
+ logger: mockLogger,
+ id: sampleSignalId,
+ signalsIndex: DEFAULT_SIGNALS_INDEX,
+ name: 'rule-name',
+ createdBy: 'elastic',
+ updatedBy: 'elastic',
+ interval: '5m',
+ enabled: true,
+ });
+ expect(successfulsingleBulkCreate).toEqual(true);
+ });
+ test('create successful bulk create with docs with no versioning', async () => {
+ const fakeUuid = uuid.v4();
+ const sampleParams = sampleSignalAlertParams(undefined);
+ const sampleSearchResult = sampleDocSearchResultsNoSortIdNoVersion;
+ mockService.callCluster.mockReturnValueOnce({
+ took: 100,
+ errors: false,
+ items: [
+ {
+ fakeItemValue: 'fakeItemKey',
+ },
+ ],
+ });
+ const successfulsingleBulkCreate = await singleBulkCreate({
+ someResult: sampleSearchResult(fakeUuid),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -125,13 +266,13 @@ describe('utils', () => {
interval: '5m',
enabled: true,
});
- expect(successfulSingleBulkIndex).toEqual(true);
+ expect(successfulsingleBulkCreate).toEqual(true);
});
- test('create unsuccessful bulk index due to empty search results', async () => {
+ test('create unsuccessful bulk create due to empty search results', async () => {
const sampleParams = sampleSignalAlertParams(undefined);
const sampleSearchResult = sampleEmptyDocSearchResults;
mockService.callCluster.mockReturnValue(false);
- const successfulSingleBulkIndex = await singleBulkIndex({
+ const successfulsingleBulkCreate = await singleBulkCreate({
someResult: sampleSearchResult,
signalParams: sampleParams,
services: mockService,
@@ -144,18 +285,15 @@ describe('utils', () => {
interval: '5m',
enabled: true,
});
- expect(successfulSingleBulkIndex).toEqual(true);
+ expect(successfulsingleBulkCreate).toEqual(true);
});
- test('create unsuccessful bulk index due to bulk index errors', async () => {
- // need a sample search result, sample signal params, mock service, mock logger
+ test('create successful bulk create when bulk create has errors', async () => {
+ const fakeUuid = uuid.v4();
const sampleParams = sampleSignalAlertParams(undefined);
const sampleSearchResult = sampleDocSearchResultsNoSortId;
- mockService.callCluster.mockReturnValue({
- took: 100,
- errors: true,
- });
- const successfulSingleBulkIndex = await singleBulkIndex({
- someResult: sampleSearchResult,
+ mockService.callCluster.mockReturnValue(sampleBulkCreateDuplicateResult);
+ const successfulsingleBulkCreate = await singleBulkCreate({
+ someResult: sampleSearchResult(fakeUuid),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -168,7 +306,7 @@ describe('utils', () => {
enabled: true,
});
expect(mockLogger.error).toHaveBeenCalled();
- expect(successfulSingleBulkIndex).toEqual(false);
+ expect(successfulsingleBulkCreate).toEqual(true);
});
});
describe('singleSearchAfter', () => {
@@ -182,6 +320,7 @@ describe('utils', () => {
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
+ pageSize: 1,
})
).rejects.toThrow('Attempted to search after with empty sort id');
});
@@ -194,6 +333,7 @@ describe('utils', () => {
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
+ pageSize: 1,
});
expect(searchAfterResult).toEqual(sampleDocSearchResultsWithSortId);
});
@@ -209,14 +349,15 @@ describe('utils', () => {
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
+ pageSize: 1,
})
).rejects.toThrow('Fake Error');
});
});
- describe('searchAfterAndBulkIndex', () => {
+ describe('searchAfterAndBulkCreate', () => {
test('if successful with empty search results', async () => {
const sampleParams = sampleSignalAlertParams(undefined);
- const result = await searchAfterAndBulkIndex({
+ const result = await searchAfterAndBulkCreate({
someResult: sampleEmptyDocSearchResults,
signalParams: sampleParams,
services: mockService,
@@ -228,12 +369,14 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(mockService.callCluster).toHaveBeenCalledTimes(0);
expect(result).toEqual(true);
});
test('if successful iteration of while loop with maxDocs', async () => {
- const sampleParams = sampleSignalAlertParams(10);
+ const sampleParams = sampleSignalAlertParams(30);
+ const someGuids = Array.from({ length: 13 }).map(x => uuid.v4());
mockService.callCluster
.mockReturnValueOnce({
took: 100,
@@ -244,7 +387,7 @@ describe('utils', () => {
},
],
})
- .mockReturnValueOnce(repeatedSearchResultsWithSortId(4))
+ .mockReturnValueOnce(repeatedSearchResultsWithSortId(3, 1, someGuids.slice(0, 3)))
.mockReturnValueOnce({
took: 100,
errors: false,
@@ -254,7 +397,7 @@ describe('utils', () => {
},
],
})
- .mockReturnValueOnce(repeatedSearchResultsWithSortId(4))
+ .mockReturnValueOnce(repeatedSearchResultsWithSortId(3, 1, someGuids.slice(3, 6)))
.mockReturnValueOnce({
took: 100,
errors: false,
@@ -264,8 +407,8 @@ describe('utils', () => {
},
],
});
- const result = await searchAfterAndBulkIndex({
- someResult: repeatedSearchResultsWithSortId(4),
+ const result = await searchAfterAndBulkCreate({
+ someResult: repeatedSearchResultsWithSortId(3, 1, someGuids.slice(6, 9)),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -276,18 +419,17 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(mockService.callCluster).toHaveBeenCalledTimes(5);
expect(result).toEqual(true);
});
- test('if unsuccessful first bulk index', async () => {
+ test('if unsuccessful first bulk create', async () => {
+ const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
const sampleParams = sampleSignalAlertParams(10);
- mockService.callCluster.mockReturnValue({
- took: 100,
- errors: true, // will cause singleBulkIndex to return false
- });
- const result = await searchAfterAndBulkIndex({
- someResult: repeatedSearchResultsWithSortId(4),
+ mockService.callCluster.mockReturnValue(sampleBulkCreateDuplicateResult);
+ const result = await searchAfterAndBulkCreate({
+ someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -298,13 +440,14 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(mockLogger.error).toHaveBeenCalled();
expect(result).toEqual(false);
});
- test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids', async () => {
+ test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids', async () => {
const sampleParams = sampleSignalAlertParams(undefined);
-
+ const someUuid = uuid.v4();
mockService.callCluster.mockReturnValueOnce({
took: 100,
errors: false,
@@ -314,8 +457,8 @@ describe('utils', () => {
},
],
});
- const result = await searchAfterAndBulkIndex({
- someResult: sampleDocSearchResultsNoSortId,
+ const result = await searchAfterAndBulkCreate({
+ someResult: sampleDocSearchResultsNoSortId(someUuid),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -326,12 +469,14 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(mockLogger.error).toHaveBeenCalled();
expect(result).toEqual(false);
});
- test('if unsuccessful iteration of searchAfterAndBulkIndex due to empty sort ids and 0 total hits', async () => {
+ test('if unsuccessful iteration of searchAfterAndBulkCreate due to empty sort ids and 0 total hits', async () => {
const sampleParams = sampleSignalAlertParams(undefined);
+ const someUuid = uuid.v4();
mockService.callCluster.mockReturnValueOnce({
took: 100,
errors: false,
@@ -341,8 +486,8 @@ describe('utils', () => {
},
],
});
- const result = await searchAfterAndBulkIndex({
- someResult: sampleDocSearchResultsNoSortIdNoHits,
+ const result = await searchAfterAndBulkCreate({
+ someResult: sampleDocSearchResultsNoSortIdNoHits(someUuid),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -353,11 +498,14 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(result).toEqual(true);
});
test('if successful iteration of while loop with maxDocs and search after returns results with no sort ids', async () => {
const sampleParams = sampleSignalAlertParams(10);
+ const oneGuid = uuid.v4();
+ const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
mockService.callCluster
.mockReturnValueOnce({
took: 100,
@@ -368,9 +516,9 @@ describe('utils', () => {
},
],
})
- .mockReturnValueOnce(sampleDocSearchResultsNoSortId);
- const result = await searchAfterAndBulkIndex({
- someResult: repeatedSearchResultsWithSortId(4),
+ .mockReturnValueOnce(sampleDocSearchResultsNoSortId(oneGuid));
+ const result = await searchAfterAndBulkCreate({
+ someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -381,11 +529,13 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(result).toEqual(true);
});
test('if successful iteration of while loop with maxDocs and search after returns empty results with no sort ids', async () => {
const sampleParams = sampleSignalAlertParams(10);
+ const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
mockService.callCluster
.mockReturnValueOnce({
took: 100,
@@ -397,8 +547,8 @@ describe('utils', () => {
],
})
.mockReturnValueOnce(sampleEmptyDocSearchResults);
- const result = await searchAfterAndBulkIndex({
- someResult: repeatedSearchResultsWithSortId(4),
+ const result = await searchAfterAndBulkCreate({
+ someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -409,41 +559,13 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(result).toEqual(true);
});
- test('if logs error when iteration is unsuccessful when bulk index results in a failure', async () => {
- const sampleParams = sampleSignalAlertParams(5);
- mockService.callCluster
- .mockReturnValueOnce({
- // first bulk insert
- took: 100,
- errors: false,
- items: [
- {
- fakeItemValue: 'fakeItemKey',
- },
- ],
- })
- .mockReturnValueOnce(sampleDocSearchResultsWithSortId); // get some more docs
- const result = await searchAfterAndBulkIndex({
- someResult: repeatedSearchResultsWithSortId(4),
- signalParams: sampleParams,
- services: mockService,
- logger: mockLogger,
- id: sampleSignalId,
- signalsIndex: DEFAULT_SIGNALS_INDEX,
- name: 'rule-name',
- createdBy: 'elastic',
- updatedBy: 'elastic',
- interval: '5m',
- enabled: true,
- });
- expect(mockLogger.error).toHaveBeenCalled();
- expect(result).toEqual(true);
- });
test('if returns false when singleSearchAfter throws an exception', async () => {
const sampleParams = sampleSignalAlertParams(10);
+ const someGuids = Array.from({ length: 4 }).map(x => uuid.v4());
mockService.callCluster
.mockReturnValueOnce({
took: 100,
@@ -454,9 +576,11 @@ describe('utils', () => {
},
],
})
- .mockRejectedValueOnce(Error('Fake Error'));
- const result = await searchAfterAndBulkIndex({
- someResult: repeatedSearchResultsWithSortId(4),
+ .mockImplementation(() => {
+ throw Error('Fake Error');
+ });
+ const result = await searchAfterAndBulkCreate({
+ someResult: repeatedSearchResultsWithSortId(4, 1, someGuids),
signalParams: sampleParams,
services: mockService,
logger: mockLogger,
@@ -467,6 +591,7 @@ describe('utils', () => {
updatedBy: 'elastic',
interval: '5m',
enabled: true,
+ pageSize: 1,
});
expect(result).toEqual(false);
});
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts
index 25934dc9aa356a9..f2a342465594573 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/alerts/utils.ts
@@ -3,6 +3,7 @@
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
+import { createHash } from 'crypto';
import { performance } from 'perf_hooks';
import { pickBy } from 'lodash/fp';
import { SignalHit, Signal } from '../../types';
@@ -59,7 +60,6 @@ export const buildRule = ({
severity: signalParams.severity,
tags: signalParams.tags,
type: signalParams.type,
- size: signalParams.size,
to: signalParams.to,
enabled,
filters: signalParams.filters,
@@ -121,7 +121,7 @@ export const buildBulkBody = ({
return signalHit;
};
-interface SingleBulkIndexParams {
+interface SingleBulkCreateParams {
someResult: SignalSearchResponse;
signalParams: AlertTypeParams;
services: AlertServices;
@@ -135,8 +135,18 @@ interface SingleBulkIndexParams {
enabled: boolean;
}
+export const generateId = (
+ docIndex: string,
+ docId: string,
+ version: string,
+ ruleId: string
+): string =>
+ createHash('sha256')
+ .update(docIndex.concat(docId, version, ruleId))
+ .digest('hex');
+
// Bulk Index documents.
-export const singleBulkIndex = async ({
+export const singleBulkCreate = async ({
someResult,
signalParams,
services,
@@ -148,15 +158,29 @@ export const singleBulkIndex = async ({
updatedBy,
interval,
enabled,
-}: SingleBulkIndexParams): Promise => {
+}: SingleBulkCreateParams): Promise => {
if (someResult.hits.hits.length === 0) {
return true;
}
+ // index documents after creating an ID based on the
+ // source documents' originating index, and the original
+ // document _id. This will allow two documents from two
+ // different indexes with the same ID to be
+ // indexed, and prevents us from creating any updates
+ // to the documents once inserted into the signals index,
+ // while preventing duplicates from being added to the
+ // signals index if rules are re-run over the same time
+ // span. Also allow for versioning.
const bulkBody = someResult.hits.hits.flatMap(doc => [
{
- index: {
+ create: {
_index: signalsIndex,
- _id: doc._id,
+ _id: generateId(
+ doc._index,
+ doc._id,
+ doc._version ? doc._version.toString() : '',
+ signalParams.ruleId ?? ''
+ ),
},
},
buildBulkBody({ doc, signalParams, id, name, createdBy, updatedBy, interval, enabled }),
@@ -171,8 +195,27 @@ export const singleBulkIndex = async ({
logger.debug(`individual bulk process time took: ${time2 - time1} milliseconds`);
logger.debug(`took property says bulk took: ${firstResult.took} milliseconds`);
if (firstResult.errors) {
- logger.error(`[-] bulkResponse had errors: ${JSON.stringify(firstResult.errors, null, 2)}`);
- return false;
+ // go through the response status errors and see what
+ // types of errors they are, count them up, and log them.
+ const errorCountMap = firstResult.items.reduce((acc: { [key: string]: number }, item) => {
+ if (item.create.error) {
+ const responseStatusKey = item.create.status.toString();
+ acc[responseStatusKey] = acc[responseStatusKey] ? acc[responseStatusKey] + 1 : 1;
+ }
+ return acc;
+ }, {});
+ /*
+ the logging output below should look like
+ {'409': 55}
+ which is read as "there were 55 counts of 409 errors returned from bulk create"
+ */
+ logger.error(
+ `[-] bulkResponse had errors with response statuses:counts of...\n${JSON.stringify(
+ errorCountMap,
+ null,
+ 2
+ )}`
+ );
}
return true;
};
@@ -182,6 +225,7 @@ interface SingleSearchAfterParams {
signalParams: AlertTypeParams;
services: AlertServices;
logger: Logger;
+ pageSize: number;
}
// utilize search_after for paging results into bulk.
@@ -190,6 +234,7 @@ export const singleSearchAfter = async ({
signalParams,
services,
logger,
+ pageSize,
}: SingleSearchAfterParams): Promise => {
if (searchAfterSortId == null) {
throw Error('Attempted to search after with empty sort id');
@@ -200,7 +245,7 @@ export const singleSearchAfter = async ({
from: signalParams.from,
to: signalParams.to,
filter: signalParams.filter,
- size: signalParams.size ? signalParams.size : 1000,
+ size: pageSize,
searchAfterSortId,
});
const nextSearchAfterResult: SignalSearchResponse = await services.callCluster(
@@ -214,7 +259,7 @@ export const singleSearchAfter = async ({
}
};
-interface SearchAfterAndBulkIndexParams {
+interface SearchAfterAndBulkCreateParams {
someResult: SignalSearchResponse;
signalParams: AlertTypeParams;
services: AlertServices;
@@ -226,10 +271,11 @@ interface SearchAfterAndBulkIndexParams {
updatedBy: string;
interval: string;
enabled: boolean;
+ pageSize: number;
}
// search_after through documents and re-index using bulk endpoint.
-export const searchAfterAndBulkIndex = async ({
+export const searchAfterAndBulkCreate = async ({
someResult,
signalParams,
services,
@@ -241,13 +287,14 @@ export const searchAfterAndBulkIndex = async ({
updatedBy,
interval,
enabled,
-}: SearchAfterAndBulkIndexParams): Promise => {
+ pageSize,
+}: SearchAfterAndBulkCreateParams): Promise => {
if (someResult.hits.hits.length === 0) {
return true;
}
logger.debug('[+] starting bulk insertion');
- const firstBulkIndexSuccess = await singleBulkIndex({
+ await singleBulkCreate({
someResult,
signalParams,
services,
@@ -260,18 +307,15 @@ export const searchAfterAndBulkIndex = async ({
interval,
enabled,
});
- if (!firstBulkIndexSuccess) {
- logger.error('First bulk index was unsuccessful');
- return false;
- }
-
const totalHits =
typeof someResult.hits.total === 'number' ? someResult.hits.total : someResult.hits.total.value;
// maxTotalHitsSize represents the total number of docs to
- // query for. If maxSignals is present we will only query
- // up to max signals - otherwise use the value
- // from track_total_hits.
- const maxTotalHitsSize = signalParams.maxSignals ? signalParams.maxSignals : totalHits;
+ // query for, no matter the size of each individual page of search results.
+ // If the total number of hits for the overall search result is greater than
+ // maxSignals, default to requesting a total of maxSignals, otherwise use the
+ // totalHits in the response from the searchAfter query.
+ const maxTotalHitsSize =
+ totalHits >= signalParams.maxSignals ? signalParams.maxSignals : totalHits;
// number of docs in the current search result
let hitsSize = someResult.hits.hits.length;
@@ -295,6 +339,7 @@ export const searchAfterAndBulkIndex = async ({
signalParams,
services,
logger,
+ pageSize, // maximum number of docs to receive per search result.
});
if (searchAfterResult.hits.hits.length === 0) {
return true;
@@ -308,7 +353,7 @@ export const searchAfterAndBulkIndex = async ({
}
sortId = sortIds[0];
logger.debug('next bulk index');
- const bulkSuccess = await singleBulkIndex({
+ await singleBulkCreate({
someResult: searchAfterResult,
signalParams,
services,
@@ -322,14 +367,11 @@ export const searchAfterAndBulkIndex = async ({
enabled,
});
logger.debug('finished next bulk index');
- if (!bulkSuccess) {
- logger.error('[-] bulk index failed but continuing');
- }
} catch (exc) {
logger.error(`[-] search_after and bulk threw an error ${exc}`);
return false;
}
}
- logger.debug(`[+] completed bulk index of ${totalHits}`);
+ logger.debug(`[+] completed bulk index of ${maxTotalHitsSize}`);
return true;
};
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts
index 7b6559561c78339..fa8fd66ef2aef6a 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/create_signals_route.ts
@@ -49,7 +49,6 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = {
risk_score: riskScore,
name,
severity,
- size,
tags,
to,
type,
@@ -92,7 +91,6 @@ export const createCreateSignalsRoute: Hapi.ServerRoute = {
riskScore,
name,
severity,
- size,
tags,
to,
type,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts
index 5e5f37ca8a08021..6639dc6a3dfd657 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.test.ts
@@ -174,7 +174,7 @@ describe('schemas', () => {
).toBeTruthy();
});
- test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score] does not validate', () => {
+ test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score] does validate', () => {
expect(
createSignalsSchema.validate>({
rule_id: 'rule-1',
@@ -190,7 +190,7 @@ describe('schemas', () => {
query: 'some query',
language: 'kuery',
}).error
- ).toBeTruthy();
+ ).toBeFalsy();
});
test('[rule_id, description, from, to, index, name, severity, interval, type, query, language, risk_score, output_index] does validate', () => {
@@ -213,7 +213,7 @@ describe('schemas', () => {
).toBeFalsy();
});
- test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score] does not validate', () => {
+ test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score] does validate', () => {
expect(
createSignalsSchema.validate>({
rule_id: 'rule-1',
@@ -228,7 +228,7 @@ describe('schemas', () => {
filter: {},
risk_score: 50,
}).error
- ).toBeTruthy();
+ ).toBeFalsy();
});
test('[rule_id, description, from, to, index, name, severity, interval, type, filter, risk_score, output_index] does validate', () => {
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts
index fa773b684eb5db5..177e7cbebc21304 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/schemas.ts
@@ -5,6 +5,7 @@
*/
import Joi from 'joi';
+import { DEFAULT_MAX_SIGNALS } from '../../../../common/constants';
/* eslint-disable @typescript-eslint/camelcase */
const description = Joi.string();
@@ -68,7 +69,7 @@ export const createSignalsSchema = Joi.object({
from: from.required(),
rule_id,
immutable: immutable.default(false),
- index: index.required(),
+ index,
interval: interval.default('5m'),
query: Joi.when('type', {
is: 'query',
@@ -95,7 +96,7 @@ export const createSignalsSchema = Joi.object({
otherwise: Joi.forbidden(),
}),
}),
- output_index: output_index.required(),
+ output_index,
saved_id: saved_id.when('type', {
is: 'saved_query',
then: Joi.required(),
@@ -103,7 +104,7 @@ export const createSignalsSchema = Joi.object({
}),
meta,
risk_score: risk_score.required(),
- max_signals: max_signals.default(100),
+ max_signals: max_signals.default(DEFAULT_MAX_SIGNALS),
name: name.required(),
severity: severity.required(),
tags: tags.default([]),
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts
index 274c41f65a36bef..1dc54f34bd1f7b9 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/update_signals_route.ts
@@ -47,7 +47,6 @@ export const createUpdateSignalsRoute: Hapi.ServerRoute = {
risk_score: riskScore,
name,
severity,
- size,
tags,
to,
type,
@@ -84,7 +83,6 @@ export const createUpdateSignalsRoute: Hapi.ServerRoute = {
riskScore,
name,
severity,
- size,
tags,
to,
type,
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts
index 22dd7be5fbba71b..ed9e00735c70417 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.test.ts
@@ -37,7 +37,6 @@ describe('utils', () => {
query: 'user.name: root or user.name: admin',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -65,7 +64,6 @@ describe('utils', () => {
query: 'user.name: root or user.name: admin',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -95,7 +93,6 @@ describe('utils', () => {
name: 'Detect Root/Admin Users',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -125,7 +122,6 @@ describe('utils', () => {
name: 'Detect Root/Admin Users',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -153,7 +149,6 @@ describe('utils', () => {
name: 'Detect Root/Admin Users',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -184,7 +179,6 @@ describe('utils', () => {
query: 'user.name: root or user.name: admin',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -215,7 +209,6 @@ describe('utils', () => {
query: 'user.name: root or user.name: admin',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -297,7 +290,6 @@ describe('utils', () => {
query: 'user.name: root or user.name: admin',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
@@ -335,7 +327,6 @@ describe('utils', () => {
query: 'user.name: root or user.name: admin',
references: ['http://www.example.com', 'https://ww.example.com'],
severity: 'high',
- size: 1,
updated_by: 'elastic',
tags: [],
to: 'now',
diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts
index e3a677741efca29..7b9921b0375d8c4 100644
--- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts
+++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/routes/utils.ts
@@ -50,7 +50,6 @@ export const transformAlertToSignal = (signal: SignalAlertType): Partial {
- return feature.properties.__kbn__isvisible__;
+ return feature.properties.__kbn_isvisibleduetojoin__;
});
expect(visibilitiesOfFeatures).to.eql([false, true, true, true]);
@@ -166,7 +166,7 @@ export default function ({ getPageObjects, getService }) {
const vectorSource = mapboxStyle.sources[VECTOR_SOURCE_ID];
const visibilitiesOfFeatures = vectorSource.data.features.map(feature => {
- return feature.properties.__kbn__isvisible__;
+ return feature.properties.__kbn_isvisibleduetojoin__;
});
expect(visibilitiesOfFeatures).to.eql([false, true, false, false]);
diff --git a/x-pack/test/functional/apps/maps/mapbox_styles.js b/x-pack/test/functional/apps/maps/mapbox_styles.js
index 49519b530337e62..bfa4be2b067af40 100644
--- a/x-pack/test/functional/apps/maps/mapbox_styles.js
+++ b/x-pack/test/functional/apps/maps/mapbox_styles.js
@@ -15,7 +15,7 @@ export const MAPBOX_STYLES = {
'all',
[
'==',
- ['get', '__kbn__isvisible__'],
+ ['get', '__kbn_isvisibleduetojoin__'],
true
],
[
@@ -89,7 +89,7 @@ export const MAPBOX_STYLES = {
'all',
[
'==',
- ['get', '__kbn__isvisible__'],
+ ['get', '__kbn_isvisibleduetojoin__'],
true
],
[
@@ -160,7 +160,7 @@ export const MAPBOX_STYLES = {
'all',
[
'==',
- ['get', '__kbn__isvisible__'],
+ ['get', '__kbn_isvisibleduetojoin__'],
true
],
[