diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c0fdc7cc005..ced899eca71 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -359,6 +359,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add ELB fileset to AWS module. {pull}14020[14020] - Add module for MISP (Malware Information Sharing Platform). {pull}13805[13805] - Add `source.bytes` and `source.packets` for uni-directional netflow events. {pull}14111[14111] +- Add Kibana Dashboard for MISP module. {pull}14147[14147] *Heartbeat* - Add non-privileged icmp on linux and darwin(mac). {pull}13795[13795] {issue}11498[11498] diff --git a/filebeat/docs/images/kibana-misp.png b/filebeat/docs/images/kibana-misp.png new file mode 100644 index 00000000000..a02068ddf30 Binary files /dev/null and b/filebeat/docs/images/kibana-misp.png differ diff --git a/filebeat/docs/modules/misp.asciidoc b/filebeat/docs/modules/misp.asciidoc index 3639bb82207..528415f6195 100644 --- a/filebeat/docs/modules/misp.asciidoc +++ b/filebeat/docs/modules/misp.asciidoc @@ -15,11 +15,22 @@ beta[] This is a filebeat module for reading threat intel information from the MISP platform (https://www.circl.lu/doc/misp/). It uses the httpjson input to access the MISP REST API interface. The configuration in the config.yml file uses the following format: - var.api_key: specifies the API key to access MISP. - var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute". - var.url: URI of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch" + * var.api_key: specifies the API key to access MISP. + * var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute". + * var.url: URI of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch" +[float] +=== Example dashboard + +This module comes with a sample dashboard. For example: + +[role="screenshot"] +image::./images/kibana-misp.png[] + +:has-dashboards!: + +:modulename!: [float] diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 88f0ca866dd..1dd0337e817 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -409,11 +409,6 @@ filebeat.modules: threat: enabled: true - # MISP Configuration - var.api_key: "XXXXXXXXXXXXXX" # API key for MISP - var.json_objects_array: "response.Attribute" - var.url: "http://X.X.X.X/attributes/restSearch" # X.X.X.X is the IP address of the MISP server - #------------------------------- Mongodb Module ------------------------------- #- module: mongodb # Logs diff --git a/x-pack/filebeat/module/misp/README.md b/x-pack/filebeat/module/misp/README.md new file mode 100644 index 00000000000..5dd8b462ab3 --- /dev/null +++ b/x-pack/filebeat/module/misp/README.md @@ -0,0 +1,27 @@ +# MISP module + +## Caveats + +* Module is to be considered _beta_. + +## How to try the module from distribution install + +You should already have MISP installed and running. Information about the MISP platform can be found here: https://www.circl.lu/doc/misp. + +``` +./filebeat setup --modules=misp -e --dashboards +``` + +Enable the MISP module + +``` +./filebeat modules enable misp +``` + +Start Filebeat + +``` +./filebeat -e +``` + +You can see the MISP Overview dashboard and the imported threat indicators in Kibana. diff --git a/x-pack/filebeat/module/misp/_meta/config.yml b/x-pack/filebeat/module/misp/_meta/config.yml index 063d5e8c7f7..edd701a7394 100644 --- a/x-pack/filebeat/module/misp/_meta/config.yml +++ b/x-pack/filebeat/module/misp/_meta/config.yml @@ -1,8 +1,3 @@ - module: misp threat: enabled: true - - # MISP Configuration - var.api_key: "XXXXXXXXXXXXXX" # API key for MISP - var.json_objects_array: "response.Attribute" - var.url: "http://X.X.X.X/attributes/restSearch" # X.X.X.X is the IP address of the MISP server diff --git a/x-pack/filebeat/module/misp/_meta/docs.asciidoc b/x-pack/filebeat/module/misp/_meta/docs.asciidoc index 5f364eda538..f1b88d76f42 100644 --- a/x-pack/filebeat/module/misp/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/misp/_meta/docs.asciidoc @@ -10,8 +10,19 @@ beta[] This is a filebeat module for reading threat intel information from the MISP platform (https://www.circl.lu/doc/misp/). It uses the httpjson input to access the MISP REST API interface. The configuration in the config.yml file uses the following format: - var.api_key: specifies the API key to access MISP. - var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute". - var.url: URI of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch" + * var.api_key: specifies the API key to access MISP. + * var.json_objects_array: specifies the array object in MISP response, e.g., "response.Attribute". + * var.url: URI of the MISP REST API, e.g., "http://x.x.x.x/attributes/restSearch" +[float] +=== Example dashboard + +This module comes with a sample dashboard. For example: + +[role="screenshot"] +image::./images/kibana-misp.png[] + +:has-dashboards!: + +:modulename!: diff --git a/x-pack/filebeat/module/misp/_meta/images/kibana-misp.png b/x-pack/filebeat/module/misp/_meta/images/kibana-misp.png new file mode 100644 index 00000000000..1f13779c702 Binary files /dev/null and b/x-pack/filebeat/module/misp/_meta/images/kibana-misp.png differ diff --git a/x-pack/filebeat/module/misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json b/x-pack/filebeat/module/misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json new file mode 100644 index 00000000000..909548471d0 --- /dev/null +++ b/x-pack/filebeat/module/misp/_meta/kibana/7/dashboard/Filebeat-MISP-Overview.json @@ -0,0 +1,417 @@ +{ + "objects": [ + { + "attributes": { + "description": "Overview dashboard for Filebeat MSIP module.", + "hits": 0, + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": {}, + "gridData": { + "h": 15, + "i": "1", + "w": 24, + "x": 0, + "y": 0 + }, + "panelIndex": "1", + "panelRefName": "panel_0", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": {}, + "gridData": { + "h": 15, + "i": "2", + "w": 24, + "x": 24, + "y": 0 + }, + "panelIndex": "2", + "panelRefName": "panel_1", + "version": "8.0.0-SNAPSHOT" + }, + { + "embeddableConfig": { + "mapCenter": [ + 24.686952411999155, + 12.128906250000002 + ], + "mapZoom": 3 + }, + "gridData": { + "h": 24, + "i": "3", + "w": 48, + "x": 0, + "y": 15 + }, + "panelIndex": "3", + "panelRefName": "panel_2", + "version": "8.0.0-SNAPSHOT" + } + ], + "timeRestore": false, + "title": "[Filebeat MISP] Overview", + "version": 1 + }, + "id": "c6cac9e0-f105-11e9-9a88-690b10c8ee99", + "migrationVersion": { + "dashboard": "7.0.0" + }, + "references": [ + { + "id": "8fc4b140-ed36-11e9-9a88-690b10c8ee99", + "name": "panel_0", + "type": "visualization" + }, + { + "id": "7d369390-f105-11e9-9a88-690b10c8ee99", + "name": "panel_1", + "type": "visualization" + }, + { + "id": "73287f70-f1fb-11e9-9a88-690b10c8ee99", + "name": "panel_2", + "type": "visualization" + } + ], + "type": "dashboard", + "updated_at": "2019-10-19T18:56:55.244Z", + "version": "WzM2NCwxNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Threat Indicator Type [Filebeat MISP]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "field": "misp.threat_indicator.type", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 20 + }, + "schema": "segment", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTooltip": true, + "isDonut": true, + "labels": { + "last_level": true, + "show": false, + "truncate": 100, + "values": true + }, + "legendPosition": "right", + "type": "pie" + }, + "title": "Threat Indicator Type [Filebeat MISP]", + "type": "pie" + } + }, + "id": "8fc4b140-ed36-11e9-9a88-690b10c8ee99", + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-10-19T19:03:26.386Z", + "version": "WzM2NSwxNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Threat Indicators per Month [Filebeat MISP]", + "uiStateJSON": {}, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": { + "customLabel": "Indicators Per Month" + }, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "customInterval": "2h", + "drop_partials": false, + "extended_bounds": {}, + "field": "@timestamp", + "interval": "M", + "min_doc_count": 1, + "timeRange": { + "from": "now-15M", + "to": "now" + }, + "time_zone": "America/Los_Angeles", + "useNormalizedEsInterval": true + }, + "schema": "segment", + "type": "date_histogram" + }, + { + "enabled": true, + "id": "3", + "params": { + "field": "misp.threat_indicator.type", + "missingBucket": false, + "missingBucketLabel": "Missing", + "order": "desc", + "orderBy": "1", + "otherBucket": false, + "otherBucketLabel": "Other", + "size": 10 + }, + "schema": "group", + "type": "terms" + } + ], + "params": { + "addLegend": true, + "addTimeMarker": false, + "addTooltip": true, + "categoryAxes": [ + { + "id": "CategoryAxis-1", + "labels": { + "show": true, + "truncate": 100 + }, + "position": "bottom", + "scale": { + "type": "linear" + }, + "show": true, + "style": {}, + "title": {}, + "type": "category" + } + ], + "grid": { + "categoryLines": false + }, + "legendPosition": "right", + "seriesParams": [ + { + "data": { + "id": "1", + "label": "Indicators Per Month" + }, + "drawLinesBetweenPoints": true, + "mode": "stacked", + "show": "true", + "showCircles": true, + "type": "histogram", + "valueAxis": "ValueAxis-1" + } + ], + "times": [], + "type": "histogram", + "valueAxes": [ + { + "id": "ValueAxis-1", + "labels": { + "filter": false, + "rotate": 0, + "show": true, + "truncate": 100 + }, + "name": "LeftAxis-1", + "position": "left", + "scale": { + "mode": "normal", + "type": "linear" + }, + "show": true, + "style": {}, + "title": { + "text": "Indicators Per Month" + }, + "type": "value" + } + ] + }, + "title": "Threat Indicators per Month [Filebeat MISP]", + "type": "histogram" + } + }, + "id": "7d369390-f105-11e9-9a88-690b10c8ee99", + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-10-19T19:03:46.399Z", + "version": "WzM2NiwxNV0=" + }, + { + "attributes": { + "description": "", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index", + "query": { + "language": "kuery", + "query": "" + } + } + }, + "title": "Threat Indicator Geo Map [Filebeat MISP]", + "uiStateJSON": { + "mapCenter": [ + -0.17578097424708533, + 0 + ], + "mapZoom": 2 + }, + "version": 1, + "visState": { + "aggs": [ + { + "enabled": true, + "id": "1", + "params": {}, + "schema": "metric", + "type": "count" + }, + { + "enabled": true, + "id": "2", + "params": { + "autoPrecision": true, + "field": "source.geo.location", + "isFilteredByCollar": true, + "mapBounds": { + "bottom_right": { + "lat": -42.68243539838622, + "lon": 60.99609375000001 + }, + "top_left": { + "lat": 42.35854391749705, + "lon": -60.99609375000001 + } + }, + "mapCenter": { + "lat": -0.17578097424708533, + "lon": 0 + }, + "mapZoom": 4, + "precision": 2, + "useGeocentroid": true + }, + "schema": "segment", + "type": "geohash_grid" + } + ], + "params": { + "addTooltip": true, + "colorSchema": "Yellow to Red", + "heatClusterSize": 1.5, + "isDesaturated": true, + "legendPosition": "bottomright", + "mapCenter": [ + 0, + 0 + ], + "mapType": "Scaled Circle Markers", + "mapZoom": 2, + "wms": { + "enabled": false, + "options": { + "format": "image/png", + "transparent": true + }, + "selectedTmsLayer": { + "attribution": "\u003cp\u003e\u0026#169; \u003ca href=\"https://www.openstreetmap.org/copyright\"\u003eOpenStreetMap contributors\u003c/a\u003e|\u003ca href=\"https://openmaptiles.org\"\u003eOpenMapTiles\u003c/a\u003e|\u003ca href=\"https://www.maptiler.com\"\u003eMapTiler\u003c/a\u003e|\u003ca href=\"https://www.elastic.co/elastic-maps-service\"\u003eElastic Maps Service\u003c/a\u003e\u003c/p\u003e\u0026#10;", + "id": "road_map", + "maxZoom": 18, + "minZoom": 0, + "origin": "elastic_maps_service" + } + } + }, + "title": "Threat Indicator Geo Map [Filebeat MISP]", + "type": "tile_map" + } + }, + "id": "73287f70-f1fb-11e9-9a88-690b10c8ee99", + "references": [ + { + "id": "filebeat-*", + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern" + } + ], + "type": "visualization", + "updated_at": "2019-10-19T19:04:04.029Z", + "version": "WzM2NywxNV0=" + } + ], + "version": "8.0.0-SNAPSHOT" +} diff --git a/x-pack/filebeat/module/misp/module.yml b/x-pack/filebeat/module/misp/module.yml new file mode 100644 index 00000000000..d3b327c7eb5 --- /dev/null +++ b/x-pack/filebeat/module/misp/module.yml @@ -0,0 +1,3 @@ +dashboards: +- id: c6cac9e0-f105-11e9-9a88-690b10c8ee99 + file: Filebeat-MISP-Overview.json diff --git a/x-pack/filebeat/module/misp/threat/ingest/pipeline.json b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json new file mode 100644 index 00000000000..0d710feeb24 --- /dev/null +++ b/x-pack/filebeat/module/misp/threat/ingest/pipeline.json @@ -0,0 +1,26 @@ +{ + "description": "Pipeline for normalizing MISP threat", + "processors": [ + { + "geoip": { + "field": "destination.ip", + "target_field": "destination.geo", + "ignore_missing": true + } + }, + { + "geoip": { + "field": "source.ip", + "target_field": "source.geo", + "ignore_missing": true + } + } + ], + "on_failure" : [{ + "set" : { + "field" : "error.message", + "value" : "{{ _ingest.on_failure_message }}" + } + }] + } + \ No newline at end of file diff --git a/x-pack/filebeat/module/misp/threat/manifest.yml b/x-pack/filebeat/module/misp/threat/manifest.yml index 2615c510833..a30ca6f626c 100644 --- a/x-pack/filebeat/module/misp/threat/manifest.yml +++ b/x-pack/filebeat/module/misp/threat/manifest.yml @@ -18,7 +18,7 @@ var: - name: interval default: 0 - name: json_objects_array - default: "" + default: "response.Attribute" - name: pagination default: |- { @@ -31,4 +31,9 @@ var: - name: url default: "" -input: config/input.yml \ No newline at end of file +input: config/input.yml +ingest_pipeline: ingest/pipeline.json + +requires.processors: +- name: geoip + plugin: ingest-geoip diff --git a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json index f76571df752..67e35fd4bed 100644 --- a/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json +++ b/x-pack/filebeat/module/misp/threat/test/misp-test.json.log-expected.json @@ -1,6 +1,13 @@ [ { "@timestamp": "2017-03-30T12:54:26.000Z", + "destination.geo.city_name": "State College", + "destination.geo.continent_name": "North America", + "destination.geo.country_iso_code": "US", + "destination.geo.location.lat": 40.7849, + "destination.geo.location.lon": -77.847, + "destination.geo.region_iso_code": "US-PA", + "destination.geo.region_name": "Pennsylvania", "destination.ip": "98.235.162.24", "event.category": "threat-intel", "event.dataset": "misp.threat", diff --git a/x-pack/filebeat/modules.d/misp.yml.disabled b/x-pack/filebeat/modules.d/misp.yml.disabled index eb2818d1b88..bd0ffb47217 100644 --- a/x-pack/filebeat/modules.d/misp.yml.disabled +++ b/x-pack/filebeat/modules.d/misp.yml.disabled @@ -4,8 +4,3 @@ - module: misp threat: enabled: true - - # MISP Configuration - var.api_key: "XXXXXXXXXXXXXX" # API key for MISP - var.json_objects_array: "response.Attribute" - var.url: "http://X.X.X.X/attributes/restSearch" # X.X.X.X is the IP address of the MISP server