From 6d89566eb378abb57afc04bee1b20332a7ba8084 Mon Sep 17 00:00:00 2001 From: Adrian Serrano Date: Mon, 28 Jun 2021 11:25:49 +0200 Subject: [PATCH 01/18] Add Recorded Future support to threatintel module (#26481) This adds a new fileset, `recordedfuture`, to the treatintel module. It ingests indicators via the Recorded Future Connect API. --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 171 ++++++ filebeat/docs/modules/threatintel.asciidoc | 105 +++- filebeat/tests/system/test_modules.py | 1 + x-pack/filebeat/filebeat.reference.yml | 32 ++ .../module/threatintel/_meta/config.yml | 32 ++ .../module/threatintel/_meta/docs.asciidoc | 105 +++- .../module/threatintel/_meta/fields.yml | 5 + x-pack/filebeat/module/threatintel/fields.go | 2 +- .../recordedfuture/_meta/fields.yml | 76 +++ .../recordedfuture/config/config.yml | 65 +++ .../recordedfuture/ingest/pipeline.yml | 236 +++++++++ .../threatintel/recordedfuture/manifest.yml | 19 + .../recordedfuture/test/domain.ndjson.log | 10 + .../test/domain.ndjson.log-expected.json | 312 +++++++++++ .../recordedfuture/test/hash.ndjson.log | 10 + .../test/hash.ndjson.log-expected.json | 494 ++++++++++++++++++ .../recordedfuture/test/ip.ndjson.log | 10 + .../test/ip.ndjson.log-expected.json | 414 +++++++++++++++ .../recordedfuture/test/url.ndjson.log | 10 + .../test/url.ndjson.log-expected.json | 442 ++++++++++++++++ .../modules.d/threatintel.yml.disabled | 32 ++ 22 files changed, 2567 insertions(+), 17 deletions(-) create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/_meta/fields.yml create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/manifest.yml create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log-expected.json create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log-expected.json create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log-expected.json create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log create mode 100644 x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 105956aa9de..5e09e0908ab 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -834,6 +834,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350] - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457] - Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441] +- Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index c983f91aca6..099296a099e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -152208,6 +152208,17 @@ example: Montreal -- +*`threatintel.indicator.geo.continent_name`*:: ++ +-- +Name of the continent. + +type: keyword + +example: North America + +-- + *`threatintel.indicator.geo.country_iso_code`*:: + -- @@ -153643,6 +153654,166 @@ type: keyword -- +[float] +=== recordedfuture + +Fields for Recorded Future Threat Intel + + + +[float] +=== entity + +Entity that represents a threat. + + + +*`threatintel.recordedfuture.entity.id`*:: ++ +-- +Entity ID. + + +type: keyword + +example: ip:192.0.2.13 + +-- + +*`threatintel.recordedfuture.entity.name`*:: ++ +-- +Entity name. Value for the entity. + + +type: keyword + +example: 192.0.2.13 + +-- + +*`threatintel.recordedfuture.entity.type`*:: ++ +-- +Entity type. + + +type: keyword + +example: IpAddress + +-- + +*`threatintel.recordedfuture.intelCard`*:: ++ +-- +Link to the Recorded Future Intelligence Card for to this indicator. + + +type: keyword + +-- + +*`threatintel.recordedfuture.ip_range`*:: ++ +-- +Range of IPs for this indicator. + + +type: ip_range + +example: 192.0.2.0/16 + +-- + +[float] +=== risk + +Risk fields. + + + +*`threatintel.recordedfuture.risk.criticality`*:: ++ +-- +Risk criticality (0-4). + + +type: byte + +-- + +*`threatintel.recordedfuture.risk.criticalityLabel`*:: ++ +-- +Risk criticality label. One of None, Unusual, Suspicious, Malicious, Very Malicious. + + +type: keyword + +-- + +*`threatintel.recordedfuture.risk.evidenceDetails`*:: ++ +-- +Risk's evidence details. + + +type: flattened + +-- + +*`threatintel.recordedfuture.risk.score`*:: ++ +-- +Risk score (0-99). + + +type: short + +-- + +*`threatintel.recordedfuture.risk.riskString`*:: ++ +-- +Number of Risk Rules observed as a factor of total number of rules. + + +type: keyword + +example: 1/54 + +-- + +*`threatintel.recordedfuture.risk.riskSummary`*:: ++ +-- +Risk summary. + + +type: keyword + +example: 1 of 54 Risk Rules currently observed. + +-- + +*`threatintel.recordedfuture.risk.riskSummary.text`*:: ++ +-- +type: text + +-- + +*`threatintel.recordedfuture.risk.rules`*:: ++ +-- +Number of rules observed. + + +type: long + +-- + [[exported-fields-tomcat]] == Apache Tomcat fields diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc index f39a9377fb1..fd3be0d93f3 100644 --- a/filebeat/docs/modules/threatintel.asciidoc +++ b/filebeat/docs/modules/threatintel.asciidoc @@ -29,6 +29,7 @@ The available filesets are: * <>: Supports gathering threat intel attributes from AlientVault OTX. * <>: Supports gathering threat intel attributes from Anomali Limo. * <>: Supports gathering threat intel attributes from Anomali ThreatStream. +* <>: Supports gathering threat intel attributes from Recorded Future. include::../include/gs-link.asciidoc[] @@ -223,7 +224,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -297,7 +298,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time the {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -409,7 +410,7 @@ Anomali Threat Intel is mapped to the following ECS fields. To configure the ThreatStream integration you first need to define an output in the Anomali ThreatStream Integrator using the Elastic SDK provided by Anomali. -It will deliver indicators via HTTP or HTTPS to a Filebeat instance running as +It will deliver indicators via HTTP or HTTPS to a {beatname_uc} instance running as a server. Configure an Integrator output with the following settings: @@ -419,12 +420,12 @@ Configure an Integrator output with the following settings: Adjust the paths to the python executable and the directory where the Elastic SDK has been unpacked. * Metadata in JSON Format: `{"url": "https://filebeat:8080/", "server_certificate": "/path/to/cert.pem", "secret": "my secret"}`. - - `url`: Use the host and port where Filebeat will be running, and `http` or `https` accordingly. + - `url`: Use the host and port where {beatname_uc} will be running, and `http` or `https` accordingly. - `server_certificate`: If using HTTPS, absolute path to the server certificate. Otherwise don't set this field. - - `secret`: A shared secret string to authenticate messages between the SDK and Filebeat. + - `secret`: A shared secret string to authenticate messages between the SDK and {beatname_uc}. -Then configure the `anomalithreatstream` fileset in Filebeat accordingly: +Then configure the `anomalithreatstream` fileset in {beatname_uc} accordingly: [source,yaml] ---- - module: threatintel @@ -449,11 +450,11 @@ Port number to use for the HTTP server. *`var.secret`*:: -Shared secret between the SDK and Filebeat, used to authenticate messages. +Shared secret between the SDK and {beatname_uc}, used to authenticate messages. *`var.ssl_certificate`*:: -Path to the public SSL certificate for the HTTPS server. If unset, Filebeat +Path to the public SSL certificate for the HTTPS server. If unset, {beatname_uc} will use unsecure HTTP connections. *`var.ssl_key`*:: @@ -488,6 +489,94 @@ Anomali ThreatStream fields are mapped to the following ECS fields: [[a]] [small]#[1]: Field is used to derive a value for the ECS field but its original value is kept under `threatintel.anomalithreatstream`.# +[[recordedfuture]] +[float] +==== `recordedfuture` fileset settings + +The `recordedfuture` fileset fetches intelligence from the Recorded Future Connect API. +It supports `domain`, `hash`, `ip` and `url` data types. + +To enable it you need to define the URL to fetch data from. You can construct this URL +using the https://api.recordedfuture.com/index.html[Recorded Future API Explorer.] The URL +must point to the `/search` endpoint and contain a suitable `limit` +(how many records to return from a single request) and `fields` parameters. +The `entity` and `timestamps` fields are required. + +Sample configuration: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +To fetch threat intelligence from multiple data types, you must define more than +one instance of the module: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 1m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +*`var.url`*:: + +The URL of the API endpoint to connect with. + +*`var.api_token`*:: + +The API token used to access Recorded Future API. + +*`var.interval`*:: + +How often the API is polled for updated information. + +*`var.first_interval`*:: + +How far back to search when retrieving events the first time {beatname_uc} starts up. +After the first interval has passed the module itself will use the timestamp +from the last response as the filter when retrieving new events. + +*`var.proxy_url`*:: + +Optional URL to use as HTTP proxy. + + +Recorded Future fields are mapped to the following ECS fields: + +[options="header"] +|============================================================= +| Recorded Future fields | ECS Fields +| entity.name | threatintel.indicator.{url,ip,domain,file.hash} +| entity.type | threatintel.indicator.type +| fileHashes | threatintel.indicator.file.hash +| intelCard | event.reference +| location.asn | threatintel.indicator.as.number +| location.location | threatintel.indicator.geo +| location.organization | threatintel.indicator.as.organization.name +| risk.score | event.risk_score +| timestamps.firstSeen | threatintel.indicator.first_seen +| timestamps.lastSeen | threatintel.indicator.last_seen +|============================================================= + :has-dashboards!: [float] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index fa8507a5952..13f9ac899fc 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -279,6 +279,7 @@ def clean_keys(obj): "threatintel.anomali", "threatintel.anomalithreatstream", "threatintel.malwarebazaar", + "threatintel.recordedfuture", "snyk.vulnerabilities", "snyk.audit", "awsfargate.log", diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index f55994e9c83..ecbcc538f16 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2274,6 +2274,38 @@ filebeat.modules: # var.ssl_certificate: path/to/server_ssl_cert.pem # var.ssl_key: path/to/ssl_key.pem + recordedfuture: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The interval to poll the API for updates + var.interval: 5m + + # How far back in time to start fetching intelligence when run for the + # first time. Value must be in hours. Default: 168h (1 week). + var.first_interval: 168h + + # The URL used for Threat Intel API calls. + # Must include the `limit` parameter and at least `entity` and `timestamps` fields. + # See the Connect API Explorer for a list of possible parameters. + # + # For `ip` entities: + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `domain` entities: + # var.url: "https://api.recordedfuture.com/v2/domain/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `hash` entities: + # var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + + # For `url` entities: + # var.url: "https://api.recordedfuture.com/v2/url/search?limit=200&fields=entity,timestamps,risk&metadata=false" + + # Set your API Token. + var.api_token: "" + #---------------------------- Apache Tomcat Module ---------------------------- - module: tomcat log: diff --git a/x-pack/filebeat/module/threatintel/_meta/config.yml b/x-pack/filebeat/module/threatintel/_meta/config.yml index ce5b5271472..f2cf00bcf0d 100644 --- a/x-pack/filebeat/module/threatintel/_meta/config.yml +++ b/x-pack/filebeat/module/threatintel/_meta/config.yml @@ -137,3 +137,35 @@ # # var.ssl_certificate: path/to/server_ssl_cert.pem # var.ssl_key: path/to/ssl_key.pem + + recordedfuture: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The interval to poll the API for updates + var.interval: 5m + + # How far back in time to start fetching intelligence when run for the + # first time. Value must be in hours. Default: 168h (1 week). + var.first_interval: 168h + + # The URL used for Threat Intel API calls. + # Must include the `limit` parameter and at least `entity` and `timestamps` fields. + # See the Connect API Explorer for a list of possible parameters. + # + # For `ip` entities: + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `domain` entities: + # var.url: "https://api.recordedfuture.com/v2/domain/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `hash` entities: + # var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + + # For `url` entities: + # var.url: "https://api.recordedfuture.com/v2/url/search?limit=200&fields=entity,timestamps,risk&metadata=false" + + # Set your API Token. + var.api_token: "" diff --git a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc index 177f5264601..3c23bdc287e 100644 --- a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc @@ -24,6 +24,7 @@ The available filesets are: * <>: Supports gathering threat intel attributes from AlientVault OTX. * <>: Supports gathering threat intel attributes from Anomali Limo. * <>: Supports gathering threat intel attributes from Anomali ThreatStream. +* <>: Supports gathering threat intel attributes from Recorded Future. include::../include/gs-link.asciidoc[] @@ -218,7 +219,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -292,7 +293,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time the {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -404,7 +405,7 @@ Anomali Threat Intel is mapped to the following ECS fields. To configure the ThreatStream integration you first need to define an output in the Anomali ThreatStream Integrator using the Elastic SDK provided by Anomali. -It will deliver indicators via HTTP or HTTPS to a Filebeat instance running as +It will deliver indicators via HTTP or HTTPS to a {beatname_uc} instance running as a server. Configure an Integrator output with the following settings: @@ -414,12 +415,12 @@ Configure an Integrator output with the following settings: Adjust the paths to the python executable and the directory where the Elastic SDK has been unpacked. * Metadata in JSON Format: `{"url": "https://filebeat:8080/", "server_certificate": "/path/to/cert.pem", "secret": "my secret"}`. - - `url`: Use the host and port where Filebeat will be running, and `http` or `https` accordingly. + - `url`: Use the host and port where {beatname_uc} will be running, and `http` or `https` accordingly. - `server_certificate`: If using HTTPS, absolute path to the server certificate. Otherwise don't set this field. - - `secret`: A shared secret string to authenticate messages between the SDK and Filebeat. + - `secret`: A shared secret string to authenticate messages between the SDK and {beatname_uc}. -Then configure the `anomalithreatstream` fileset in Filebeat accordingly: +Then configure the `anomalithreatstream` fileset in {beatname_uc} accordingly: [source,yaml] ---- - module: threatintel @@ -444,11 +445,11 @@ Port number to use for the HTTP server. *`var.secret`*:: -Shared secret between the SDK and Filebeat, used to authenticate messages. +Shared secret between the SDK and {beatname_uc}, used to authenticate messages. *`var.ssl_certificate`*:: -Path to the public SSL certificate for the HTTPS server. If unset, Filebeat +Path to the public SSL certificate for the HTTPS server. If unset, {beatname_uc} will use unsecure HTTP connections. *`var.ssl_key`*:: @@ -483,6 +484,94 @@ Anomali ThreatStream fields are mapped to the following ECS fields: [[a]] [small]#[1]: Field is used to derive a value for the ECS field but its original value is kept under `threatintel.anomalithreatstream`.# +[[recordedfuture]] +[float] +==== `recordedfuture` fileset settings + +The `recordedfuture` fileset fetches intelligence from the Recorded Future Connect API. +It supports `domain`, `hash`, `ip` and `url` data types. + +To enable it you need to define the URL to fetch data from. You can construct this URL +using the https://api.recordedfuture.com/index.html[Recorded Future API Explorer.] The URL +must point to the `/search` endpoint and contain a suitable `limit` +(how many records to return from a single request) and `fields` parameters. +The `entity` and `timestamps` fields are required. + +Sample configuration: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +To fetch threat intelligence from multiple data types, you must define more than +one instance of the module: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 1m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +*`var.url`*:: + +The URL of the API endpoint to connect with. + +*`var.api_token`*:: + +The API token used to access Recorded Future API. + +*`var.interval`*:: + +How often the API is polled for updated information. + +*`var.first_interval`*:: + +How far back to search when retrieving events the first time {beatname_uc} starts up. +After the first interval has passed the module itself will use the timestamp +from the last response as the filter when retrieving new events. + +*`var.proxy_url`*:: + +Optional URL to use as HTTP proxy. + + +Recorded Future fields are mapped to the following ECS fields: + +[options="header"] +|============================================================= +| Recorded Future fields | ECS Fields +| entity.name | threatintel.indicator.{url,ip,domain,file.hash} +| entity.type | threatintel.indicator.type +| fileHashes | threatintel.indicator.file.hash +| intelCard | event.reference +| location.asn | threatintel.indicator.as.number +| location.location | threatintel.indicator.geo +| location.organization | threatintel.indicator.as.organization.name +| risk.score | event.risk_score +| timestamps.firstSeen | threatintel.indicator.first_seen +| timestamps.lastSeen | threatintel.indicator.last_seen +|============================================================= + :has-dashboards!: [float] diff --git a/x-pack/filebeat/module/threatintel/_meta/fields.yml b/x-pack/filebeat/module/threatintel/_meta/fields.yml index 48222e31ebd..fce6811f5fe 100644 --- a/x-pack/filebeat/module/threatintel/_meta/fields.yml +++ b/x-pack/filebeat/module/threatintel/_meta/fields.yml @@ -167,6 +167,11 @@ ignore_above: 1024 description: City name. example: Montreal + - name: continent_name + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America - name: country_iso_code type: keyword ignore_above: 1024 diff --git a/x-pack/filebeat/module/threatintel/fields.go b/x-pack/filebeat/module/threatintel/fields.go index de37293b142..a0b33b519a2 100644 --- a/x-pack/filebeat/module/threatintel/fields.go +++ b/x-pack/filebeat/module/threatintel/fields.go @@ -19,5 +19,5 @@ func init() { // AssetThreatintel returns asset data. // This is the base64 encoded gzipped contents of module/threatintel. func AssetThreatintel() string { - return "eJzsfW1z2ziS//v5FF3+v4i9JSm2Y2cnrsr/ymM7E9U5TtYPM1t7uVIgsiViDQIMAErWXt13v8IDKT6Jki0qN7e1fpNIBLt/aDQaje4G1IdHXJyBjiQSTblG9hOAppph9UuJDInCMxijJj8BhKgCSRNNBT+D//8TAMC9fQHsG4xOkQcIHyjDsfn2kwhThoOfACYUWajO7Ct94CSu8TJ/IU5IyvTItj6DCWEK/SO9SPAMplKkSd64Bsb8fbCcYCJFDDrCIpclsDgHZv6K4IoAKQ9pQLSQgwmVSo8UIs8bZZAecTEXMix8vwKYkxZCSDQC4SFoGiPMI+Rl6SmRygDBsgSJiZAaQ1B0GmnKp6AjqgrIWkAzsgqzgdA5YMNuK7zZK6qGlwk+3QzvTRqPUYKYWLCqwh3mRIEYK5QzDCEQPEwDD9JqMQk0nVG9aENpEG2pBIsEDcIlLKKM4CQq5EZ24wVcLEw3PluoZMwQKIe7++Ff4XhwOChRu3pKMDBvzQhLUZWeAfwJSKoFF7FIVV8tlMa43kJqOiGBrj0IqcRAC7moPxExobxvZFN7hjGhrE/CUNYeTSirt6fJ7KS5OU1mb5ufxCRY8SDV+FT7NpEiQFWXjRITPSeyjimVrP6dQtknQSBSXhfVnPJQzFVf4pQqLRf9R6xL7al/eviuH6CRtxl5bFGzgv5sp22X9tHYzgW0BIzuGU0XPJsCTueW1rJ1lgaEc5QjpYneZqZeGDkaKOe/vb66vIUZ8lBIg5JoUGlgBmySMraAELXT8JgwGlCRKqtIICQ83F63YU2kmNEQ5XYSHIbIzYh5ERom1sBEWLaDGbc2RIHgExqa5p1iWpIFSaxFI0rRKV8ObAYOUmUeW1tSeEsFhKF6nmW5ERruEgwMjLAHN4JjD67FvAefMKRp3IOPdBrVXjvsHx3WvjwPYyoJo3oBdwYK7B/13x7Uml3eDLPnp/13p/UGv199yRoM40QoRY3x7MMFSk0oP2gZGucT7ERVlJNS4P0OtyqFRBOgCgIRmxExDkub5pjmCvVu8XkmS++pYaFvA0mTGr7SV5tBI9l6XFohCYfhFzBWH5WCfSolGtyazmwf3GJFBT9olaJduDoT4gqkfn18KUjjQm1hWTfAZzgAd77SC0HaZX7gh2PXAuXOq9h29GMiHymfDjSr6+nzXDhJJma+XBuXFb5IoUUgGKiISDOXPZ92ewpE4lndgEVU1/2RX2XZjXffnpvxq317i2GrCHQQYVjrfnFnBQ17oiUtokVMgxLbVRJslWHNGjnChZG3roAHDASYCAgD5DMqBY+Ra0AeJoIaN0ICRz0X8hFwhlwPGnDbHu0AtqWbOQQ77kNl89FVFzKvcGc9qOshqduMzVXQWa4GQVSMZEUKD5x+TzGze4SZrpj5qIXreb5TArdTGtjtb+3r3HCe391UPZDUsmALoEsJIwmiXDCCW15DrlFy1GULgU8kThiewdHp0dt3DR0Xcko4/Qcx/RnUdl+rlYFOuZA4ImMxM9QPj09Kj+OUaTqqy7ugePhU3fQ4bg0PuJCxKkdvGgbjc6ErlskKUfwqxJQhXF9ftGhTtu3aQqeM6zNQWlYCEFtItXUGXgiuzeSx4ZS5pNZpd+wtx8raAfBFJCmz6uoWRCIlWTS+bh1Lr9OZYAbwQUgwrn9tB+7fylo67rB/e/Xr6O5vPTD/Xv31y/nN5ejubwc957uqSKQshDEWkFBd9fUFR0/ds8fvqXEklfU5HVvzmuXx6eH6fmg5Wg4ZUcZgXEU8I5LaoAhDPtWRI87TGKV3YXtmAxkZQRnKl79/vr20ASzz6S/mU6kbFepjhCSXtYVnBBliQGPClnEap7j7OJjCt72jvW8HK/T31X/sXZx9lZp8lRiOtE6+jin/Gi9IkgzwCff+81WDMiakIsxulPBDypil3QPKA5aGZgQiOsOeIW1FZH2T5p58/PfrT1/vPn+4//389urrJxpIocREf/3dxT7g5v7rRSolcv0bSkUF/zqMydSFg+HqCYO0Es0wf58tNPV1TrnpmhHJ10scp9NpycBngqnD60YyN4VNveVhJ5VGvmJUWyBWIz/dALzNpqcZqLIU6uZwimILSxhQvRh1t7iUunFhdvgt1v6T4FoiYU2wRMq1XIyoEqNAhDtB51jA8O4zGBYrQF6ct8DbleA8tBbZXRBOQtIAzfo51anntQLFyDpvqzlfCz6lOg1dDoARbT+ssnb/BXtM8L0z6P/5zeDt0cnPbw57sMeI3juDk9PB6eHpu6Of4b+bjJ5ZgQTf5fjeWg7rh7f/l4vV+HY0wB5by/j+JcUxBi3TfkIZDhIc0DiJiIo22OK2AizBe3UOhmYe+IwTIbUCyoHAlysbkB3AOQfPG/p9sx1wzaCCBszTgHCz1KbKOeATyqcoE2n2EWPKibSe8ww5kIlGCRIDESeUuWVXSBA6QmmHsc9whuV4vZaEq4mQsW2uICIzBBEEZnkKezCPaBDB3PowQUT4FCEWEs1rITVvEOZ663bq5fG4RiK5a080RFon6uz16/l8PphQibjAQSDi12Mmpq9dLKNvHAYig+j18eHRyevDo9dakuCR8mk/JmxOJPadnPqGp/GWIh2zQXGK5DpwGLz9+fBNcILvjo+PzH/CgJy+e/uGkPDN2zCcrNGOLVaF2hg2E2giUdhGMFV1ElfPnjUetEtOml69UpmiGfo9oBMgM0KZ8REHjTiUChGTnSBxpK24NkESh6c7gRGHpxtjUBE52o0sInL0HBTHp293heP49O1zkLz5+WRXSN78fPIcJKdHx7tCcnp0vAbJC+JO22yKM3iWfBMORf/RhKM97LOKyysFWmjCLNUmbs9Y6TdlWF3eM1b4pJGrZh9tG3453SamMY1x1FVIscD00/DTVWUI68tROcX+7GhNNY/zYtyXLlXj/ZpUsmXwYM8s6siI0jQYBGKvi4Hbfnogs6UrmlCeBaAZLnEsc3dC0inlNmbxPUWlG9BPJJnGWPH+dwP+i5DOdcsF7T078+nb//tWELsWSaOsJyljXQz5cGJJwcPttc3CeO+BcG080YVIpXFLISAKewbeohDvUlpIrFpdyuFbKtnAUP1m3Eu0zqmNMbkBo8p6sFxp6eothAQfSDJvGxnYkHmFcDXjWowEu8HtQh4PPBahzeIvdcaOjwKFttprCbABkvm7ERpdpoDyPNYdC061kJRPe04hs8qrh9triMnCxg/zobByk0iqhQZmi2GrQYCJqXKUDAGqQEw0cvh76grP8vopl+IkOqqivC8NSIx+xPN3c9pEAdWlarEemP0HQ23rTrhoTM8kRKma6Hc0nTyrbD75Od6MauNg4hqWOqqwK0za126P0zhxKwntlcu227Gd+djw82zLSmQnJ2+aMH1PUe4iUNdktC0vr3hhqR7LPfGx8KYeVKgZSX9/79ekBolnHL/92zej4vgUsDTEcLkoFBkOjCUkVuHzBYUL866dZbW6OGofFzpjCdiW5hmxXMepzlv1Cixd7/GJKl2d4TbIbnP8iV5is91w7b95GpWYQUgnE5TINSUaYYx6Xs+S2/TmXFhjrpr0wOU9UGI46s6pMNgjOo3QWqaMgTWqjknPdjNJMJ/AKh27R9Xh/CBktv/vFXI6lqCvM5kICXsTIQa+3SAQ8Z4Zkr3iF43W0AW7vWBD1ChjyjE0i1NAFbKFHx1gVGlg9BFdOVk6ZjQAlU4mtFpvaVvuR1onZ69fu4au3UDI6cEA7uXCZrkEkCSR4onGRPuaqPECFI0TtgBNHqsmwA2mLTM2I8rIGJlyKSIuNNglZ46MWXHcX1+qpXEKxCB9bDRNKohwJ/G8qkrcWUar7acNJDVDzDTjB1mrnJ+1295nc+v0Ar6nhDlXwbexNWQum1SrESSMZR02zaw9wsQts5FQ2r2c8tC7gbW5OAAYcruYS00JY9Wa2iqang06TnwpK2bPXfgbbJwwA2TdDss/IJyLqudVmg29gkxyS1nr3BiZmDfP0JY5XZ78RZG77QdRehAvPBmnx25mE6VrU9qZ42xoIqJcaj+xudKZmS5ismRW0D6Vjo8HKh0flUxIr2H+LaE6i+5dYy+WAqW9njMdXICWhDIz5xOUVISNEQaRjCzE51nhbXUdJxNfRKZF4hUkK97D++vLgx4QpgQ8cjHnRlJL8VZddWviemZscjNl1DbTkcJ0GdRtepV7hfhk2d4MjFWAfzKTbs35Kmu+HKZN7XqqUO4oU1PbPnlWK13xevDj6fTw3RbRD4WSEjZaWQa1bQ99oZRjk5U7UaXSZT154RQDkFRHQlLtq0zMNteYPx4sqgbEmuZcRY3PyJKI+NKNntlzLXfabjOQFWGIVEMgmDCGl4eQJglK49NVGAQRkSTQKNWKJNrp6Ydffnl38efLq18+HL77+fDd5dHxxcV5U0bXdngX4s2KDQwDM21WyPIikwRSG064pEpTPk2pijB0RPYvbw7Mknch4lhw/93FzUEPQkyQ2/oOwRv37Mt05/uHux58fn/l16MhD3rw+eG9XX2WNqcHFzd5m7uP58e2uh/OlUolKZ9nMH93Ztcsm1PlKh3/HYNdBJ2KZRxFqXqOYPYKP1y0d/fvL4wbIiSnpAfX7+8Ihw9GaFQFoiD6npH9wApaRURiOJgyMSYsHwaOTUE8wrQxQMY82uT0LsrXrs0C4HwHK8gCT+/97N+d3xwMnJxcCdmMyIUxF00Hntxfrux2ThcHzFakju2cN+Jni9zBWJ4BQNWDy5s7qPcZYN8QnFMWBkSGyqziPCzXkFfTussShj8VYr6vWoy4olNOdCq3PDPyyeWAYUJiyhZWyG6S7RezMg117WScKvQZ5LWLSQuA7MSskHBuSF58zDH5073DwhldaF2ZJpR1l1L4kKWCYJqaEbOLz8PtdUTSwuA1GZiGkXkxitL40EUr41QyA24UijlngnRSbn7tq3dg/+H2+sAFSWEhUuvhZYyAQCCShbN91B2Qa0U6ozJVNv01kKhStqk1boV6/ps/rGfgSlelvCEIs5w3JyImTJCW8qRWDGZD5QhvioNR/tjJqFH+mBXh/paT9+ejV8zlTVJiP2ge005U1+ywhpeFlM9Gk0eijeoFnczebBy82TAegFw7h+3h1nTTdXStCAJXFQuOaiaOh9vrAXzJjigWzgSB4Ixy7IGYTMx/nMPL7Ra0FbkrN+oKtT+IFQh71Eo4j8dqNFXgl53yAdwGSGNGgkeze1QDlcpxJ+mqu4fbX66XlL1YV8jStMDQipALPXIfN0WckNiZ846Ae3pw2YS/DZS/VmHTzcjasb2fU61RQkR4yAobV8fFJfIidyuBu86hOuiwLyQQLvgiFqk6aAXPiNTYZE/GQjAkfHPoQ+d7oSqkWLEEy4AeI/ICcpGHG20EKjsHva9lanN59pzKQeu8IhsfCWlfmlwcxUibTBUQpURAy8cMvqcoqTvqnvWpvlZwERNGu1oqHLU/xhLRcE/Ibspxyof4W5gmxEyUzpJCntymXZ4RRsPRRIq4AUBY3VC1cv89Ql5maPPb7pqZiUi5rUmw59G5MjPEHVSljdHarF5gV6hsRLCBydKkjJF1tj77TJJEVjwOmCNq0v2GZ1tBcAGxqkr0ioHfPK9hNkRU5RkPa5rnWUAjP8DSNGzNl4psYc8KjzfSZ3vLVCcyM4SyXLq7wWcN68AduOuC+dWTlsSefDRTJm86w4zJSlXa5Zne+9IsMkR6dotYVKExwl6mNzYa0rOb7I9ERf27j+fHp28bg+nCBn1G/li7cdA7m3v2IpLc4/ecGvZHbpVyPqnSEkm8k/Xvrkq65UoyaFgemxSPEaXyJGQXcqu7QYQXhp4qSCSdEVeqZCs/SJbdmaB3hvPAkn/CFvUDpyV3+iwj2vNvrAif+VatEmm6BCeThoqqxUJrdShGolK5PMEeBKkkwQL2bd8PzSw8Ojw8KN2JUxzuV0ZgGFKXYSKcsIWmgQKNQcQFE9OFIZELuN1VD1ETyo4belY7Kt1el2rpOCtj9LTRhhTkPoy9wzte2OQXnBy34vxB7lsTZ4t0pFApe96rm0sZLstHk8CTd3uZwAy23cYUbaTgJTVoB93Z1QslE7120u2RRPuU+F7PfbKGO/tAk+x/qWS1hP3eWGRNguMCmeB4+a19D/boceK/iwnLm1bpmWc5f/Nh+Uocnmb/9SQTIh8x9E2SiKooe7dK1j0sNvQkVEC4/97sxgtdUP6FKimVqsTdElZorIU0ja3lMx8kcj2yp7BQWlatYx8TtmL0swzDc4Y/P5YGEqdE2rAKyTd8LgvQAwIXv13B8NJWtBDuTrsRrUnwCPZSDqO3vcY95GazsHrw6IXqbFwHa5825yzRJc1GqaRdQLg1zpax3A+3wzoUb5DXJAxwhpLqTetB2292kFTTwN1l1jQ8Pq5mV2J/2V2SMFq1TmstAxPzHsT+prWITqMezFAu+ua/7Z210u+iq3f+qtLVww/FJercrKyFcwgrsY2McLpbGJb4NE6lVYpq2rYJiq7uZF8qJUPIg2i+C7UkJXsJaav/pGWqNIajgMqA4YiGnXjjyxXU0wdH35f1x3kkcNN5niYh0djRMD5YYjC8XBe774TZ7fUzbZqdlJ0lPi/tDS6FS5k2two06eWRAXuYyW/z4vC0vqvyi86Y/IOQ+iWZL9tPZSmnXyzRP2IGuYyw+Zjhj8gf/yECzK7WoAlU8erHBnDPO/id5amrdRorDoKsPci6vOc5p+x2ul7a9bEt2qXdIPF0N8ZhJmYjiLpom4W7JPUrcpSVS/lgzVHhtT1bqm52Cmzq2IBxxUG7OxCrFimDNLzfNZrh/UoghbCRy1A1KPDzDg0XbszzcVhfpTMnClQ6jql2eUbPkDVO9ECEODLWpRNHU4RoTZXZQdDCviJbuqpze3nuV9Xvv3yhtR/effnfSh2day3pOK14BaXL8oKOvI/SZXUXIo5Tbtz7pcu08sZGIae7wLABZ7fP8EX1jRCee2zejrKrUrc27tRG1vy5TTjKSt9joTQEfhfUvLJMRCcKEOa3tWTBeFraXpeC8CsFZcOZKuomRZ0ntNxpWWMdcvqNdeppd4G4h4eKWtTC6aXrDptW9edl7/JfhxATdxNhlbPtvw++NdrnbAqP6tfqv0RHP4o5xIQvloT9AW/uz0Ha+4rcNYhrRWR/QUKTOOlETjm1FwsrpMp1qqM0wmWBnnOSCzPGGPbGySJFIlQWBRwxETSVyz172tyhtj/94XID01RiCIK75cUmsGzpruHmblClau3wmcYdlZ34gnJfxmVBlQaOKs+t7Wy4swKj7rTqXMPcbM3tz7I0KlOTZStcLSMpn47sut/RElXOBljKGDpYCvKoi1q7dIVUkTEzRkFalWxW+JeMY4GizaE7RmF2LW9taFde+RGq0Y8w3UA1xPaSccd1hYsx6A6JqLgZTatoi3AMmC4LdUT1muCXANrVSJXAvQCXvUu7QwvVYJQIKySBJcZC13/Hoewyd6hLq91myv0lfI12Ky/eK/8oRgVnl2q2Aqk/3Lw91i41cAVWq5hdYO1aK1fgtXVemYKqBQ+yayo20NPcvetQWZfTOqf+3Dm9hNVlUY8u3c6/CbTS+XIfAKbJ7KTnbuQjPLRHaNu7EBCN0+qPf21TBe/pbdGVvRt/s9C5/6G2xmqlwiiIFcmRl6ryEnPhBvRUC7PjDAhji0yRsytNhpd37RB3tTS1S7cdk23SoUvofzLCDms3CNdshZ6/aSwDyqvIcz4Y9qo/0WeziSj98ejGu2UKmrijvWTJ3i87YGyrLRaIXmK8AhE/41K4NTFKS0pBTELMf2gjx0m1QjZpR7OrjYqlV9DH7Md73PcVncx/IWuJq11BkWFXhx6q06VwxiEWs+LlTpsP8Y/ZYeVIkZe2WR0ssL4kdRfL/2fvPzdPLbqcWRshbBHvVt5ALuS6cuTJvspOcv1ce87PKzyvpj3nUTk/n1e023tI9tz9mxE58onrvpLBXj2DIXTxhpJtEhif7//6f+Doy7/OG8C/zhv8s543+J8AAAD//2Eg0rs=" + return "eJzsfd1z2ziy7/v8FV2+D7G3JMV27OzEVXNveWxnRnUdJ+uPzNbe3FIgsiViDQIMAErRnjr/+yl8kOK3ZJvK2bO1fkkkQugfuhuNRncDHMIjrs5ARxKJplwj+wlAU82w+qVEhkThGUxRk58AQlSBpImmgp/B//4JAODe/gDsLxidIw8Q3lOGU/PtBxGmDEc/AcwoslCd2Z8MgZO4Rsv8hTgjKdMT2/oMZoQp9I/0KsEzmEuRJnnjGhjz995SgpkUMegIi1TWwOIcmPkrgisCpDykAdFCjmZUKj1RiDxvlEF6xNVSyLDwfQswxy2EkGgEwkPQNEZYRsjL3FMilQGCJQkSEyE1hqDoPNKUz0FHVBWQdYBmpA2zgdA7YEPuRXizn6gaXib4fDu8N2k8RQliZsGqCnVYEgViqlAuMIRA8DANPEirxSTQdEH1qgulQfRCJVglaBCuYRFlGCdRITe8m67gYmWG8dFCJVOGQDnc3Y//Csejw1Gpt6vvCQbmVwvCUlSlZwB/ApJqwUUsUjVUK6UxrreQms5IoGsPQiox0EKu6k9ETCgfGt7UnmFMKBuSMJS1RzPK6u1psjhpbk6TxdvmJzEJWh6kGr/Xvk2kCFDVeaPETC+JrGNKJat/p1AOSRCIlNdZtaQ8FEs1lDinSsvV8BHrXPs+PD18NwzQ8NtIHjvUrKA/L9O2S/toaucC2g6M7hlNFzybAk7n1tayc5YGhHOUE6WJfslMvTB8NFDOP7++uryFBfJQSIOSaFBpYAQ2SxlbQYjaaXhMGA2oSJVVJBASHm6vu7AmUixoiPJlHByHyI3EPAsNEWtgIizbwYxaF6JA8BkNTfNeMa27BUmsRSNK0TlfCzYDB6kyj60tKfxKBYSheppluREa7hIMDIxwADeC4wCuxXIAHzCkaTyA3+k8qv3scHh0WPvyPIypJIzqFdwZKLB/NHx7UGt2eTPOnp8O353WG/xx9SlrMI4ToRQ1xnMIFyg1ofygQzTOJ9iJqijHpcD7HW5VCokmQBUEIjYSMQ5Ll+aY5gr1bvF5ImvvqWGh7wJJkxq+0lfbQSPZelxaIQmH8ScwVh+Vgn0qJRrcmi7sGNxiRQU/6OSiXbh6Y2ILUr8+PhekcaFeYFm3wGcoAHe+0jNB2mV+5MWxa4Zy51W8VPoxkY+Uz0ea1fX0aS6cJDMzX66NywqfpNAiEAxURKSZy55Otz0FIvGsbsAiquv+yG+y7Ma7b8+N/Grf3mLYyQIdRBjWhl/cWUHDnmjdF9EipkGJbBsHO3lYs0au44LkrSvgAQMBJgLCAPmCSsFj5BqQh4mgxo2QwFEvhXwEXCDXowbcdkQ7gG37zRyCHY+hsvnoawiZV7izEdT1kNRtxvYq6CxXAyMqRrLChQdOv6WY2T3CzFDMfNTCjTzfKYHbKY3s9rf2dW44z+9uqh5IakmwFdA1h5EEUc4YwS2tMdcoOeqyhcDvJE4YnsHR6dHbdw0DF3JOOP0HMeMZ1XZf7cpA51xInJCpWJjeD49PSo/jlGk6qfO7oHj4vbrpcdQaHnAhY1WO3jQI42NhKJZICyt+E2LOEK6vLzq0Kdt2vUCnjOszUlpWAhAv4GrnDLwQXJvJY8MpS0mt0+7IW4qVtQPgk0hSZtXVLYhESrJq/Ll1LL1OZ4wZwXshwbj+tR24/1XW0lGH/dur3yZ3fxuA+ffqr5/Oby4nd387GDjfVUUiZSFMsYCE6qqvLzj63j15/JYaR1JZn9ORNT+zND48XN+PLUVLIeuUMZhWES+IpDYowpDPdeQ652mM0ruwA7OBjAyjTM+Xf3y8vbQBLPPpL+ZTaRiV3qcISc5rC88wMsSAxoSt4zROcfdxNIeve0d7Xw9a9PfV/9u7OPsiNfkiMZxonXyZUv4lXpEkGeF33Pv/rxqUMSEVZvajhO9TxmzfA6A8YGloJBDRBQ5M15ZF1jdpHsnv//f6w5e7j+/v/zi/vfrygQZSKDHTX/5wsQ+4uf9ykUqJXH9GqajgX8YxmbtwMFx9xyCtRDPM30cLTX1ZUm6GZljy5RKn6XxeMvAZY+rw+uHMTWFTb2nYSaWRt0i1A2I18tMPwNtsehpBlblQN4dzFC+whAHVq0l/i0tpGBdmh99h7T8IriUS1gRLcE05cr0rbEUdyIm14LwRUkdwbk0OaQSbci1XE6rEJBDhTljpSMD47iMYEi1IL8474O1Kyh5ah6AvCCdhE+esU1a1E16FUUysp9lO+VrwOdVp6BIWjGj7oc00/wfsMcH3zmD45zejt0cnP785HMAeI3rvDE5OR6eHp++Ofob/bLLQZrkUfJfyvbUUNot3+JeLdnw7ErDH1iHfv6Q4xaDDRs0ow1GCIxonEVHRFvvxToAleK/OwfSZR2njREitgHIg8OnKRo9HcM7B04bh0OxdXDOooAHzNCDc+AWpcruFGeVzlIk0m54p5URaN3+BHMhMowSJgYgTypyPICQIHaG0YhwyXGA5uaAl4WomZGybK4jIAkEEgVlLwwEsIxpEsLQOVxARPkeIhUTzs5CaXxDmRuvCCmV5XCOR3LUnGiKtE3X2+vVyuRzNqERc4SgQ8espE/PXLvAyNN4NkUH0+vjw6OT14dFrLUnwSPl8GBO2JBKHjk9DQ9O4dpGO2ag4RXIdOAze/nz4JjjBd8fHR+Y/YUBO3719Q0j45m0YzjZoxwuWsJoMmzto6qKw52Gq6tG2z54N7r7LpJpRvVKZopn+B0BnQBaEMuPQjhpxKBUiJjtB4rq27NoGSRye7gRGHJ5ujUFF5Gg3vIjI0VNQHJ++3RWO49O3T0Hy5ueTXSF58/PJU5CcHh3vCsnp0fEGJM8Ikr1kB5/Bs9034VD0H004umNUbVReKdBCE2Z7baL2hJV+W4LV5T0jhd81ctXso72EXt5vE9GYxjjpK/5ZIPph/OGqIsL6clSuB3hyaKmadHo27kuXV/J+TSrZOtKxZxZ1ZERpGowCsdeH4F4+PZDZOhtNKM+i5QzXONaJRiHpnHIbYPmWotIN6GeSzGOseP+7Af9JSOe65Yz2np359PV/fS2wXYukkdezlLE+RD6e2a7g4fbapoy890C4Np7oSqTSuKUQEIUDA29VCM4pLSRWrS7l8DWVbGR6/WrcS7TOqQ2IOYFRZT1YrrR0xSFCgo96mV8bHtj4fqXjanq4GLZ2wu2DHw88FqEtOVjrjJWPAoW2NG0NsAGS+bsRGl1ag/I8MB8LTrWQlM8HTiGzMrGH22uIycoGO3NRWL5JJNWqCLPFsKUrwMRcuZ5MB1SBmGnk8PfUVcnlxV4uH0t0VEV5XxJIjF7i+W/zvokCqkulbQMw+w+G2hbJcNGYS0qIUjXW72g6eVLZfPJzvBnV1pHPDSR1VCFXmLSv3R6nceJWsu+ty7bbsZ35QPbTbEsrspOTN02YvqUodxFVbDLalpZXvLBUPOae+MB90wgqvRlOf/vFr0kNHM8ofv0/X42K4/eApSGG60WhSHBkLCGxCp8vKFyY39pZVivio/ZxYTC2A9vSPCOW6jTVeatBgaQbPX6nSldnuM0I2IKERK+x2WG49l99H5WYQUhnM5TINSUaYYp6WU/p21zsUlhjrpr0wCVpUGI46c+pMNgjOo/QWqaMgDWqjsjADjNJMJ/AKp26R1Vxvhcy2/8PCgko26EvipkJCXszIUa+3SgQ8Z4RyV7xi0Zr6CLznrEhapQx5RiaxSmgCtnKSwcYVRoYfURX+5ZOGQ1ApbMZrRaH2pb7kdbJ2evXrqFrNxJyfjCCe7myKTkBJEmk+E5jon0B13QFisYJW4Emj1UT4IRpa6KNRBmZIlMun8WFBrvkLJExy47760u1Nk6BGKWPjaZJBRHuJJ5XVYk7S6jdftpAUjPETDN+kLXK6Vm77X02t06v4FtKmHMVfBtb8OZSX7WCRsJYNmDTzNojTNwyGwml3Y9THno3sDYXRwBjbhdzqSlhrFoAXEUzsEHHma+7xey5C3+DjRNmgKzbYekHhHNR9bxKs2FQ4EluKWuDmyITy+YZ2jGny5O/yHK3/SBKj+KV78bpsZvZROnalHbmOBNNRJSrQ0hsYndhpouYrYkVtE+l0+ORSqdHJRMyaJh/a6jOonvX2LOl0NPewJkOLkBLQpmZ8wlKKsLGCINIJhbi06zwS3UdZzNf8aZF4hUkqzTE++vLgwEQpgQ8crHkhlNr9lZddWviBkY2uZkyapvpSGG6jOo2vUq90vls3d4IxirAv5hJt+a8zZqvxbStXU8Vyh1lamrbJ0+q1RWvBz++nx6+e0H0Q6GkhE1aa7ZeOkJf1eXIZLVZVKl0XfxeOHIBJNWRkFT7khizzTXmjwerqgGxpjlXUeMzsiQivs5kYPZc65222wxkFSMi1RAIJozh5SGkSYLS+HQVAkFEJAk0StWSRDs9ff/rr+8u/nx59ev7w3c/H767PDq+uDhvyujaAe+CvVlW3BAw06aFlxcZJ5DacMIlVZryeUpVhKHrZP/y5sAseRcijgX3313cHAwgxAS5LUYRvHHPvk53/vJwN4CPv1z59WjMgwF8fPjFrj5rmzOAi5u8zd3v58f2KAKcK5VKUj58Yf7uzK5ZNqfKVTr9Owa7CDqV6g0KXPUUwewVfjhr7+5/uTBuiJCckgFc/3JHOLw3TKMqEAXWDwzvR5bRKiISw9GciSlhuRg4NgXxCNPGABnzaJPTu6i1uzYLgPMdLCMLNL33s393fnMwcnxy9W4LIlfGXDSdznJ/ubLbOV0UmC2fndo5b9jPVrmDsT6wgGoAlzd3UB8zwL7pcElZGBAZKrOK87Bc8F5N665LGP5UiPm+6jDiis450al84QGXDy4HDDMSU7ayTHaTbL+YlWkowifTVKHPIG9cTDoAZMd7hYRz0+XF7zkmfxR5XDhQDJ0r04yy/lIK77NUEMxTIzG7+DzcXkckLQivycA0SObZKEryoatOwqlkBtwkFEvOBOmlNv7aV+/A/sPt9YELksJKpNbDywgBgUAkK2f7qDvN14l0QWWqbPprJFGlbFtr3An1/LM/WWjgSldSvSUIs5w3JyJmTJCO8qRODGZD5TreFgej/LEXqVH+mFUMf86794e5W+byNimxHzSPaS+qa3ZY48tCymerySPRRvWCXmZvJgdvNowHIDfOYXsSN912Hd3IgsCV8ILrNWPHw+31CD5l5ykLB5hAcEY5DkDMZuY/zuHldgvaidyVG/WF2p8aC4Q9Fyacx2M1mirwy075tHADpCkjwaPZPaqRSuW0l3TV3cPtr9frnj1bW3hpWmBoWciFnriP2yJOSOzMeU/AfX9w2YS/C5S/A2LbzchG2d4vqdYoISI8ZIWNq6PiEnmRu0LB3T1RFTrsCwmEC76KRaoOOsEzIjU22ZOpEAwJ3x762PleqAopVizBMqCniLyAXOThRhuByg5t72uZ2lyePVRz0DmvyNbnV7qXJhdHMdwmcwVEKRHQ8pmIbylK6s7lZ2OqrxVcxITRvpYK19s/xxLRcKnJbspxyjcOdBBNiJkovSWFfHfbDnlBGA0nMyniBgBhdUPVSf2PCHmZoM1vuztxZiLltibBHp7nyswQd6qWNkZrs3qBXaGyEcEGImuTMkXW2/rsM0kSWfHsYo6oSfcbnr0IgguIVVViUAz85nkNsyGiKs94WNO8zAIa+WmbJrE134DyAntWeLyVPtsrsXrhmekoy6W764Y2kA7c6cA+iF9915LYY5pmyuRNF5gRaVWlXR5Avi/NItPJwG4Riyo0RdjL9MZGQwZ2k/07UdHw7vfz49O3jcF0YYM+E38G3zjovc09e2tK7vF7Sg37I7dKOZ9UaYkk3sn6d1ftuuP+NGhYHpsUjxGl8iRkH3yru0GEF0RPFSSSLogrVbKVHyTL7szQO8N5YMk/Yav66diSO32WdTrwv2gJn/lWnRxpurEn44aKqsVCG3UoRqJSuT5uHwSpJMEK9u3YD80sPDo8PChd4FMU9yvDMAypyzARTthK00CBxiDigon5ynSRM7jbVQ9RE8qOG0ZWO9fdXZdq+3FWxuhpow0p8H0ce4d3urLJLzg57sT5g9y3JsoW6UShUva8Vz83SFyWjyaB797tZQIjbLuNKdpIwUtq0A26t3siSiZ646TbI4n2KfG9gftkDXf2gSbZ/1LJagn7vanImgTHhW6C4/W39newR48T/11MWN602p95ltM3H9Y/icPT7L++y4TIRwx9kySiKsp+W+3WPSw29F2ogHD/vdmNF4ag/A+qXalUJe5Ks0JjLaRpbC2f+SCR64k9hYXSkuqUfUxYi/SzDMNTxJ8fSwOJcyJtWIXkGz6XBRgAgYvPVzC+tBUthLvTbkRrEjyCvUHE6O2gcQ+53SysHjx6pjob18Hap+0pS3RJs0kqaR8Qbo2zZSz3w+24DsUb5A0JA1ygpHrbetDuaygk1TRwF681icfH1exK7G/mSxJGq9Zpo2VgYjmA2F8LF9F5NIAFytXQ/Ld7sJb7fQz1zt+r2i5+KC5R52ZlLZxDaMU2Mczpb2FY49M4l1YpqmnbJii6upN9LpdMRx5E88WtJS7ZG1M7/SctU6UxnARUBgwnNOzFG1+voL5/cP37sv44jwRuO8/TJCQaexLjg+0MxpebYve9ELu9fqJNs5Oyt8Tnpb1upnCD1PZWgSaDPDJgDzP5bV4cntZ3VX7RmZJ/EFK/0fN5+6ks5fSr7fSfMYNcRth8zPBH5I//KQLMrtagCVTxnsoGcE87+J3lqat1Gi0HQTYeZF1fSp337Ha6ntt12Rbt0m6Q+H63xmEmZiOIOmubmbvu6jfkKCs3CMKGo8IbR7ZW3ewU2NyRAeOKg3YXNlYtUgZpfL9rNOP7ViCFsJHLUDUo8NMODReu9/NxWF+lsyQKVDqNqXZ5Rk+QNU70QIQ4MdalF0dThGhNldlB0MK+Ilu6qnN7fe5X1S/rfKa1H999+u9KHZ1rLek0rXgFpZv9gp68j9LNehcijlNu3Pu1y9R6vaSQ811g2IKy22f4ovpGCE89Nm+l7KrUrY07tZE1f24TjrLS91goDYHfBTWvLDPRiwKE+W0tWTCelrbXpSB8K6NsOFNF/aSo84SWOy1rrEPef2OdetpfIO7hoaIWtXB66W7GplX9adm7/FUWYuauTaxStuP3wbdG+5xN4Un9HQDP0dHfxRJiwlfrjv0Bb+7PQdr7itydjRtZZF93oUmc9MKnvLdnMyukyg2qpzTCZaE/5yQXZowx7I2TRYpEqCwKOGEiaCqXe/K0uUNt31PicgPzVGIIgrvlxSawbOmuoeaue6Vqo/hM457KTnxBuS/jsqBKgqPKU+s6G+6swKQ/rTrXsDRbc/sOmUZlarJshatlJOXziV33e1qiytkA2zOGDpaCPOqiNi5dIVVkyoxRkFYlmxX+OXIs9Ghz6I5QmN0hXBNt65UfoZr8CNMNVENsb0R3VFtcjFF/SETFzWhaRTuYY8D0WagjqncaPwfQriRVAvcMXPbi7x4tVINRIqyQBJYYC11/6UTZZe5Rl9rdZsr9JXyNdisv3iu/waOCs081a0HqDze/HGufGtiC1SpmH1j71soWvLbOK1NQteJBdk3FFnqau3c9Kut6Wue9P3VOr2H1WdSjS68S2AZa6Xy5DwDTZHEycDfyER7aI7TdQwiIxnn1TWUvqYL3/b1gKHs3/mahc/9WucZqpYIUREty5LmqvMZcuK491cLsOAPC2CpT5OxKk/HlXTfEXS1N3dztxmSb9OgS+vdbWLH2g3DDVujpm8YyoLyKPKeD4aD6PkGbTUTpj0c33i1T0MQd7SVL9n49AGNbbbFA9BzjFYj4CZfCbYhR2q4UxCTE/K0gOU6qFbJZN5pdbVRsfwV9zN405L6v6GT+Oq81rm4FRYZ9HXqoTpfCGYdYLIqXO20v4h+zw8qRIi9ts3pYYH1J6i6W/4/ef26eWnQ9s7ZC2MHeF3kDOZPrypEn+yo7yc1z7SnvgnhaTXtOo3J+Pq9ot/eQ7Ln7NyNy5BPXQyWDvXoGQ+jiDSUvSWB8vP/r/4CjL/8+bwD/Pm/wr37eYF0fGAgZYjhLGy9XqM/zzgr97czAracJ7y3R7U1CISxo7xapMKk9ob8h3X1le3O1V7njuX6ZYzXr3lwe0GqxiuDa8vFbZORzmOUEbPaXl7Tt0eTs6N3x6HB0PDp6s9eKshbR6Rsnt7dvfbbGL8uS+zthOvFvA742r/oG31QvXoY5Ts7d5SZ7jRpqK3ouquXK3RA3wCtec1CdQ+Pii24NWcdy0VkGmWNNJpLweZWjDmrLww1Yb+2LPsQMxp/UxnLM4oUwmfQPXx+9fdWIVlL12NfMv6Xq0c/np87yYF163KqJ01XtGp4tQOXACiRg/3B4clDXyAY012SK1Von6Gdy1FDZ05wj+MitrN0rtB94qlLCBnCXnw4YwIfs3ecD+Ixytf7cPiRcuHNL7nBOtYQMCneWEK2R1y4Kf8KYXqmcWr2EvYpLBaJ2Wy90HKl6Gndt70ba7951iNvMgbv6xc1FIC8S9LrQzoK6TRmur+d2d37PSGDP9cz8ix14/hOZ1t7F7v4KNv716Um7dbejS+OYyPap1T68DVdtbc0DJw4HY8NozKhPT4q88vlrtsq5NqqPt+vNpVWeNLyotMyQjgatbzQt8qPTsStjsQJulUxDWSc8VetkSeFGP/1XAAAA///mLJ1d" } diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/_meta/fields.yml b/x-pack/filebeat/module/threatintel/recordedfuture/_meta/fields.yml new file mode 100644 index 00000000000..3e4a82a2ec2 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/_meta/fields.yml @@ -0,0 +1,76 @@ +- name: recordedfuture + type: group + default_field: false + description: > + Fields for Recorded Future Threat Intel + fields: + - name: entity + type: group + description: > + Entity that represents a threat. + fields: + - name: id + type: keyword + description: > + Entity ID. + example: "ip:192.0.2.13" + - name: name + type: keyword + description: > + Entity name. Value for the entity. + example: "192.0.2.13" + - name: type + type: keyword + description: > + Entity type. + example: "IpAddress" + - name: intelCard + type: keyword + description: > + Link to the Recorded Future Intelligence Card for to this indicator. + - name: ip_range + type: ip_range + description: > + Range of IPs for this indicator. + example: '192.0.2.0/16' + - name: risk + type: group + description: > + Risk fields. + fields: + - name: criticality + type: byte + description: > + Risk criticality (0-4). + - name: criticalityLabel + type: keyword + description: > + Risk criticality label. One of None, Unusual, Suspicious, Malicious, Very Malicious. + - name: evidenceDetails + type: flattened + description: > + Risk's evidence details. + - name: score + type: short + description: > + Risk score (0-99). + - name: riskString + type: keyword + description: > + Number of Risk Rules observed as a factor of total number of rules. + example: "1/54" + - name: riskSummary + type: keyword + ignore_above: 1024 + description: > + Risk summary. + example: "1 of 54 Risk Rules currently observed." + multi_fields: + - name: text + type: text + norms: false + default_field: false + - name: rules + type: long + description: > + Number of rules observed. diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml b/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml new file mode 100644 index 00000000000..238d55d170c --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml @@ -0,0 +1,65 @@ +{{ if eq .input "httpjson" }} + +type: httpjson +config_version: "2" +interval: {{ .interval }} + +request.method: GET +{{ if .ssl }} +request.ssl: {{ .ssl | tojson }} +{{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} +request.url: "{{ .url }}&orderby=lastseen&direction=asc" +request.transforms: +{{ if .api_token }} +- set: + target: header.X-RFToken + value: {{ .api_token }} +- set: + target: url.params.lastSeen + value: '[[ .cursor.timestamp ]]' + default: '([[ formatDate (now (parseDuration "-{{ .first_interval }}")) "2006-01-02T15:04:05.000Z" ]],]' + {{ end }} +response.split: + target: body.data.results +cursor: + timestamp: + value: '([[ .first_event.timestamps.lastSeen ]],]' + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - decode_json_fields: + fields: [message] + target: json + - fingerprint: + fields: + - event.dataset + - json.entity.id + target_field: "@metadata._id" + encoding: base64 + - add_fields: + target: '' + fields: + ecs.version: 1.10.0 + - script: + lang: javascript + id: set_opt_type + source: > + function process(event) { + event.Put("@metadata.op_type", "index"); + } diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml new file mode 100644 index 00000000000..0a5e9937ed4 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml @@ -0,0 +1,236 @@ +description: Pipeline for parsing Recorded Future threat intel. +processors: + # + # Safeguard against feeding the pipeline with documents other + # that the ones generated by Filebeat's httpjson input. + # + - fail: + if: 'ctx.json == null || !(ctx.json instanceof Map)' + message: 'missing json object in input document' + + # + # Set basic ECS fields. + # + - set: + field: event.ingested + value: '{{{ _ingest.timestamp }}}' + - set: + field: event.kind + value: enrichment + - set: + field: event.category + value: threat + - set: + field: event.type + value: indicator + + # + # Map itype field to STIX 2.0 Cyber Observable values (threatintel.indicator.type). + # + - script: + lang: painless + if: 'ctx.json.entity?.type != null' + description: > + Map entity.type field to STIX 2.0 Cyber Observable values (threatintel.indicator.type). + params: + IpAddress: ipv4-addr + InternetDomainName: domain-name + Hash: file + URL: url + source: > + String mapping = params[ctx.json.entity.type]; + if (mapping != null) { + ctx["threatintel_indicator_type"] = mapping; + } + on_failure: + - append: + field: error.message + value: 'Unable to determine indicator type from "{{{ json.entity.type }}}": {{{ _ingest.on_failure_message }}}' + + - rename: + field: threatintel_indicator_type + target_field: threatintel.indicator.type + ignore_missing: true + + # + # Detect ipv6 for ipv4-addr types. + # + - set: + field: threatintel.indicator.type + value: ipv6-addr + if: 'ctx.threatintel?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && ctx.json.entity.name.contains(":")' + + # + # Map first and last seen dates. + # + - date: + field: json.timestamps.firstSeen + target_field: threatintel.indicator.first_seen + formats: + - ISO8601 + if: 'ctx.json.timestamps?.firstSeen != null' + on_failure: + - append: + field: error.message + value: 'Error parsing firstSeen field value "{{{ json.timestamps.firstSeen }}}": {{{ _ingest.on_failure_message }}}' + - date: + field: json.timestamps.lastSeen + target_field: threatintel.indicator.last_seen + formats: + - ISO8601 + if: 'ctx.json.timestamps?.lastSeen != null' + on_failure: + - append: + field: error.message + value: 'Error parsing lastSeen field value "{{{ json.timestamps.lastSeen }}}": {{{ _ingest.on_failure_message }}}' + + + # + # Map location fields. + # + - rename: + field: json.location.location.city + target_field: threatintel.indicator.geo.city_name + ignore_missing: true + - rename: + field: json.location.location.continent + target_field: threatintel.indicator.geo.continent_name + ignore_missing: true + - rename: + field: json.location.location.country + target_field: threatintel.indicator.geo.country_name + ignore_missing: true + - grok: + field: json.location.asn + patterns: + - '^(?:[Aa][Ss])?%{NUMBER:threatintel.indicator.as.number:long}$' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Cannot parse asn field `{{{ json.location.asn }}}`: {{{ _ingest.on_failure_message }}}' + + - rename: + field: json.location.organization + target_field: threatintel.indicator.as.organization.name + ignore_missing: true + + - set: + field: event.reference + value: '{{{ json.intelCard }}}' + ignore_empty_value: true + + - set: + field: json.ip_range + copy_from: json.entity.name + if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")' + - set: + field: json.ip_range + value: '{{{ json.entity.name }}}/32' + if: 'ctx.threatintel?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")' + - set: + field: json.ip_range + value: '{{{ json.entity.name }}}/128' + if: 'ctx.threatintel?.indicator?.type == "ipv6-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")' + - set: + field: json.ip_range + copy_from: json.entity.name + if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")' + + - rename: + field: json.entity.name + target_field: threatintel.indicator.ip + if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")' + + - rename: + field: json.entity.name + target_field: threatintel.indicator.domain + ignore_missing: true + if: 'ctx.threatintel?.indicator?.type == "domain-name"' + + - uri_parts: + field: json.entity.name + target_field: threatintel.indicator.url + keep_original: true + remove_if_successful: true + if: 'ctx.threatintel?.indicator?.type == "url"' + on_failure: + - append: + field: error.message + value: 'Cannot parse url field `{{{ json.entity.name }}}`: {{{ _ingest.on_failure_message }}}' + + # At this point fileHashes may exist if "fileHashes" field is requested. + - append: + field: json.fileHashes + value: '{{{ json.entity.name }}}' + allow_duplicates: false + if: 'ctx.threatintel?.indicator?.type == "file"' + + - remove: + field: json.entity.name + if: 'ctx.threatintel?.indicator?.type == "file"' + + - script: + lang: painless + description: > + Map file hashes. + if: 'ctx.json.fileHashes != null' + params: + '4': crc32 + '32': md5 + '40': sha1 + '64': sha256 + '128': sha512 + source: > + def hashes = new HashMap(); + for (def hash : ctx.json.fileHashes) { + def algo = params[String.valueOf(hash.length())]; + if (algo != null) { + hashes[algo] = hash; + } + } + ctx["_hashes"] = hashes; + on_failure: + - append: + field: error.message + value: 'Failed to map fileHashes field: {{ _ingest.on_failure_message }}' + + - rename: + field: _hashes + target_field: threatintel.indicator.file.hash + ignore_missing: true + + # + # Map risk.score to event.risk_score. + # + - convert: + field: json.risk.score + target_field: event.risk_score + ignore_missing: true + type: float + on_failure: + - append: + field: error.message + value: 'Risk score `{{{ json.risk.score }}}` cannot be converted to float: {{ _ingest.on_failure_message }}' + # + # Remove fields converted to an ECS field. + # + - remove: + field: + - json.timestamps + - json.location + - json.fileHashes + - message + ignore_missing: true + + # + # Save fields without an ECS mapping under `threatintel.recordedfuture`. + # + - rename: + field: json + target_field: threatintel.recordedfuture + +on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/manifest.yml b/x-pack/filebeat/module/threatintel/recordedfuture/manifest.yml new file mode 100644 index 00000000000..93df3884160 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: input + default: httpjson + - name: first_interval + default: 168h + - name: interval + default: 1m + - name: url + default: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + - name: ssl + - name: tags + default: [threatintel-recordedfuture, forwarded] + - name: proxy_url + - name: api_token +ingest_pipeline: + - ingest/pipeline.yml +input: config/config.yml diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log new file mode 100644 index 00000000000..54f047c3ab6 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "idn:16url-gy.example.net", "name": "16url-gy.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A16url-gy.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-07-25T20:29:32.750Z", "lastSeen": "2021-06-20T18:23:47.901Z"}} +{"entity": {"id": "idn:b999f.example.org", "name": "b999f.example.org", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ab999f.example.org", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2012-11-21T01:54:04.292Z", "lastSeen": "2021-06-20T18:23:47.812Z"}} +{"entity": {"id": "idn:c422.example.net", "name": "c422.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac422.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2018-02-21T13:53:46.470Z", "lastSeen": "2021-06-20T18:23:47.778Z"}} +{"entity": {"id": "idn:8rwcvgjsp.example.net", "name": "8rwcvgjsp.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A8rwcvgjsp.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-08-15T11:56:24.964Z", "lastSeen": "2021-06-20T18:23:47.747Z"}} +{"entity": {"id": "idn:c9px.example.net", "name": "c9px.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac9px.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-06-29T21:06:06.066Z", "lastSeen": "2021-06-20T18:23:47.460Z"}} +{"entity": {"id": "idn:ttj1i9z7.example.com", "name": "ttj1i9z7.example.com", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Attj1i9z7.example.com", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2018-09-20T03:26:08.564Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:7pgc.example.org", "name": "7pgc.example.org", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A7pgc.example.org", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2017-02-23T17:44:16.104Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:xm5u434.example.net", "name": "xm5u434.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Axm5u434.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2017-04-10T06:55:27.658Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:gpgju.example.com", "name": "gpgju.example.com", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Agpgju.example.com", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2018-07-27T15:22:39.390Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:55g.example.com", "name": "55g.example.com", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A55g.example.com", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-01-10T21:24:38.353Z", "lastSeen": "2021-06-20T18:23:45.025Z"}} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log-expected.json new file mode 100644 index 00000000000..12d7044c9a1 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log-expected.json @@ -0,0 +1,312 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A16url-gy.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "16url-gy.example.net", + "threatintel.indicator.first_seen": "2016-07-25T20:29:32.750Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.901Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:16url-gy.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A16url-gy.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ab999f.example.org", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 482, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "b999f.example.org", + "threatintel.indicator.first_seen": "2012-11-21T01:54:04.292Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.812Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:b999f.example.org", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ab999f.example.org", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac422.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 955, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "c422.example.net", + "threatintel.indicator.first_seen": "2018-02-21T13:53:46.470Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.778Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:c422.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac422.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A8rwcvgjsp.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1425, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "8rwcvgjsp.example.net", + "threatintel.indicator.first_seen": "2016-08-15T11:56:24.964Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.747Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:8rwcvgjsp.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A8rwcvgjsp.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac9px.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1910, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "c9px.example.net", + "threatintel.indicator.first_seen": "2016-06-29T21:06:06.066Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.460Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:c9px.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac9px.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Attj1i9z7.example.com", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2380, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "ttj1i9z7.example.com", + "threatintel.indicator.first_seen": "2018-09-20T03:26:08.564Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:ttj1i9z7.example.com", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Attj1i9z7.example.com", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A7pgc.example.org", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2862, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "7pgc.example.org", + "threatintel.indicator.first_seen": "2017-02-23T17:44:16.104Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:7pgc.example.org", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A7pgc.example.org", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Axm5u434.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3332, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "xm5u434.example.net", + "threatintel.indicator.first_seen": "2017-04-10T06:55:27.658Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:xm5u434.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Axm5u434.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Agpgju.example.com", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3811, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "gpgju.example.com", + "threatintel.indicator.first_seen": "2018-07-27T15:22:39.390Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:gpgju.example.com", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Agpgju.example.com", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A55g.example.com", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4284, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "55g.example.com", + "threatintel.indicator.first_seen": "2021-01-10T21:24:38.353Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:45.025Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:55g.example.com", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A55g.example.com", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log new file mode 100644 index 00000000000..284429cc3e3 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "hash:dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "name": "dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "type": "Hash"}, "fileHashes": ["25328d1a481903f2d900479570842247", "d73c663e2ac0c7a14ca0e2681dd599b2e7a24f65", "dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Adec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://23l04ha7h.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.503Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://frj972mua.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:26.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.503Z", "lastSeen": "2021-06-20T18:40:18.503Z"}} +{"entity": {"id": "hash:4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "name": "4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "type": "Hash"}, "fileHashes": ["7b8d9afd032f0c253b7dd68aca6fb50b", "f9ece49c249aabab29fd9c2193d897b7d131ed17", "4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "8 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://poapoq2z.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.452Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://mezsa92p.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:27.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.452Z", "lastSeen": "2021-06-20T18:40:18.452Z"}} +{"entity": {"id": "hash:299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "name": "299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "type": "Hash"}, "fileHashes": ["7b65b50ed4554c86cb777e35e7750209", "e10942ba3fbb937c90c7cb3e39c06a13324981a8", "299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "9 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://kyvhpghg.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.343Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://fdxeziea.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:25.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.343Z", "lastSeen": "2021-06-20T18:40:18.343Z"}} +{"entity": {"id": "hash:e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "name": "e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "type": "Hash"}, "fileHashes": ["c6353df35499ca6934da2169b7bd1635", "3e208c649da0a9efbde7bbde6eece2142fdac3f9", "e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ae5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://k40z19-by.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.257Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://4e-6k-.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:29.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.258Z", "lastSeen": "2021-06-20T18:40:18.258Z"}} +{"entity": {"id": "hash:184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "name": "184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "type": "Hash"}, "fileHashes": ["3d568bd03766a8d47c8fabb7d392c32e", "3ea8b08bc9ed3009a4d6a0ab5851b8e3fc10ead2", "184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://ksmt6j.network.local/scan/results/file/184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:24.000Z"}], "riskString": "1/14", "riskSummary": "1 of 14 Risk Rules currently observed.", "rules": 1, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.131Z", "lastSeen": "2021-06-20T18:40:18.131Z"}} +{"entity": {"id": "hash:1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "name": "1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "type": "Hash"}, "fileHashes": ["a40e91f2d29616076114eea0f2a693af", "e38ccd47629c1b75385a83fbfbba0ea7f3b3a705", "1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "8 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://m-1z.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.093Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://llt6m.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:29.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.093Z", "lastSeen": "2021-06-20T18:40:18.093Z"}} +{"entity": {"id": "hash:bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "name": "bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "type": "Hash"}, "fileHashes": ["02062782c7eeaff185ea6966460f7c9a", "64355796dc38992ca5e434682ddbf63bdfabeb4e", "bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Abf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "4 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://46h0mn.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.070Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://j94d.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:28.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.070Z", "lastSeen": "2021-06-20T18:40:18.070Z"}} +{"entity": {"id": "hash:c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "name": "c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "type": "Hash"}, "fileHashes": ["bdd205ffc81c54e7cc1a9080cfa093e4", "a6b928fd6fee43495b96941ef80b25d074f6e0e2", "c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://5twber.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.010Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://b5qxg4.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:28.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.011Z", "lastSeen": "2021-06-20T18:40:18.011Z"}} +{"entity": {"id": "hash:c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "name": "c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "type": "Hash"}, "fileHashes": ["af45390e39574cdb037d684074e6a542", "f6a14c7424604cd51ba6a6d3f7594ec762f48645", "c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://wor2ca.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:17.964Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://l4tlgg.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:31.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:17.964Z", "lastSeen": "2021-06-20T18:40:17.964Z"}} +{"entity": {"id": "hash:0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "name": "0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "type": "Hash"}, "fileHashes": ["5b8bcd367f802cd104210bb47abb3ab1", "b40d1796bd6974860ce6be691152ad963300c711", "0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://79073cr.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:17.919Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://c2ilj.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:26.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:17.919Z", "lastSeen": "2021-06-20T18:40:17.919Z"}} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log-expected.json new file mode 100644 index 00000000000..32a800a1574 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log-expected.json @@ -0,0 +1,494 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Adec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "25328d1a481903f2d900479570842247", + "threatintel.indicator.file.hash.sha1": "d73c663e2ac0c7a14ca0e2681dd599b2e7a24f65", + "threatintel.indicator.file.hash.sha256": "dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.503Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.503Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Adec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://frj972mua.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:26.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://23l04ha7h.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.503Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1478, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "7b8d9afd032f0c253b7dd68aca6fb50b", + "threatintel.indicator.file.hash.sha1": "f9ece49c249aabab29fd9c2193d897b7d131ed17", + "threatintel.indicator.file.hash.sha256": "4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.452Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.452Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://mezsa92p.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:27.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "8 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://poapoq2z.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.452Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2954, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "7b65b50ed4554c86cb777e35e7750209", + "threatintel.indicator.file.hash.sha1": "e10942ba3fbb937c90c7cb3e39c06a13324981a8", + "threatintel.indicator.file.hash.sha256": "299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.343Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.343Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://fdxeziea.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:25.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "9 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://kyvhpghg.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.343Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Ae5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4457, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "c6353df35499ca6934da2169b7bd1635", + "threatintel.indicator.file.hash.sha1": "3e208c649da0a9efbde7bbde6eece2142fdac3f9", + "threatintel.indicator.file.hash.sha256": "e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.258Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.258Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ae5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://4e-6k-.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:29.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://k40z19-by.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.257Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 5932, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "3d568bd03766a8d47c8fabb7d392c32e", + "threatintel.indicator.file.hash.sha1": "3ea8b08bc9ed3009a4d6a0ab5851b8e3fc10ead2", + "threatintel.indicator.file.hash.sha256": "184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.131Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.131Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://ksmt6j.network.local/scan/results/file/184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:24.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/14", + "threatintel.recordedfuture.risk.riskSummary": "1 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 7054, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "a40e91f2d29616076114eea0f2a693af", + "threatintel.indicator.file.hash.sha1": "e38ccd47629c1b75385a83fbfbba0ea7f3b3a705", + "threatintel.indicator.file.hash.sha256": "1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.093Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.093Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://llt6m.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:29.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "8 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://m-1z.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.093Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Abf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 8550, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "02062782c7eeaff185ea6966460f7c9a", + "threatintel.indicator.file.hash.sha1": "64355796dc38992ca5e434682ddbf63bdfabeb4e", + "threatintel.indicator.file.hash.sha256": "bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.070Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.070Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Abf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://j94d.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:28.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "4 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://46h0mn.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.070Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 10020, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "bdd205ffc81c54e7cc1a9080cfa093e4", + "threatintel.indicator.file.hash.sha1": "a6b928fd6fee43495b96941ef80b25d074f6e0e2", + "threatintel.indicator.file.hash.sha256": "c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.011Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.011Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://b5qxg4.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:28.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://5twber.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.010Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 11492, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "af45390e39574cdb037d684074e6a542", + "threatintel.indicator.file.hash.sha1": "f6a14c7424604cd51ba6a6d3f7594ec762f48645", + "threatintel.indicator.file.hash.sha256": "c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "threatintel.indicator.first_seen": "2021-06-20T18:40:17.964Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:17.964Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://l4tlgg.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:31.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://wor2ca.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:17.964Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 12964, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "5b8bcd367f802cd104210bb47abb3ab1", + "threatintel.indicator.file.hash.sha1": "b40d1796bd6974860ce6be691152ad963300c711", + "threatintel.indicator.file.hash.sha256": "0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "threatintel.indicator.first_seen": "2021-06-20T18:40:17.919Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:17.919Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://c2ilj.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:26.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://79073cr.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:17.919Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log new file mode 100644 index 00000000000..bb05454a584 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "ip:2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", "name": "2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", "location": {"asn": "AS31287", "cidr": {"id": "ip:151.237.36.0/23", "name": "151.237.36.0/23", "type": "IpAddress"}, "location": {"city": "Radnevo", "continent": "Europe", "country": "Bulgaria"}, "organization": "IPACCT CABLE Ltd"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-04-18T00:11:48.512Z", "lastSeen": "2021-06-19T19:40:32.897Z"}} +{"entity": {"id": "ip:2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", "name": "2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", "location": {"asn": "AS197207", "cidr": {"id": "ip:93.110.128.0/17", "name": "93.110.128.0/17", "type": "IpAddress"}, "location": {"city": null, "continent": "Asia", "country": "Iran"}, "organization": "Mobile Communication Company of Iran PLC"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T17:55:58.019Z", "lastSeen": "2021-06-19T19:40:32.839Z"}} +{"entity": {"id": "ip:203.0.113.55", "name": "203.0.113.55", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.55", "location": {"asn": null, "cidr": {"id": "ip:0.0.0.0/8", "name": "0.0.0.0/8", "type": "IpAddress"}, "location": {"city": null, "continent": null, "country": null}, "organization": null}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T19:40:30.596Z", "lastSeen": "2021-06-19T19:40:30.596Z"}} +{"entity": {"id": "ip:203.0.113.108", "name": "203.0.113.108", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.108", "location": {"asn": "AS17622", "cidr": {"id": "ip:58.248.128.0/19", "name": "58.248.128.0/19", "type": "IpAddress"}, "location": {"city": "Guangzhou", "continent": "Asia", "country": "China"}, "organization": "China Unicom Guangzhou network"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T19:40:20.534Z", "lastSeen": "2021-06-19T19:40:20.534Z"}} +{"entity": {"id": "ip:203.0.113.139", "name": "203.0.113.139", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.139", "location": {"asn": "AS7713", "cidr": {"id": "ip:125.162.0.0/16", "name": "125.162.0.0/16", "type": "IpAddress"}, "location": {"city": null, "continent": "Asia", "country": "Indonesia"}, "organization": "PT Telekomunikasi Indonesia"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-06-23T07:39:06.418Z", "lastSeen": "2021-06-19T19:40:03.882Z"}} +{"entity": {"id": "ip:2001:db8:bf58:c5c3:7a06:5267:82e0:621a", "name": "2001:db8:bf58:c5c3:7a06:5267:82e0:621a", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:bf58:c5c3:7a06:5267:82e0:621a", "location": {"asn": "AS17622", "cidr": {"id": "ip:58.249.64.0/19", "name": "58.249.64.0/19", "type": "IpAddress"}, "location": {"city": "Guangzhou", "continent": "Asia", "country": "China"}, "organization": "China Unicom Guangzhou network"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T19:40:02.557Z", "lastSeen": "2021-06-19T19:40:02.557Z"}} +{"entity": {"id": "ip:192.0.2.147", "name": "192.0.2.147", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.147", "location": {"asn": "AS4837", "cidr": {"id": "ip:61.53.0.0/17", "name": "61.53.0.0/17", "type": "IpAddress"}, "location": {"city": "Zhengzhou", "continent": "Asia", "country": "China"}, "organization": "CHINA UNICOM China169 Backbone"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2017-12-20T02:21:07.734Z", "lastSeen": "2021-06-19T19:39:43.160Z"}} +{"entity": {"id": "ip:203.0.113.198", "name": "203.0.113.198", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.198", "location": {"asn": "AS9829", "cidr": {"id": "ip:59.93.20.0/22", "name": "59.93.20.0/22", "type": "IpAddress"}, "location": {"city": "Palakkad", "continent": "Asia", "country": "India"}, "organization": "National Internet Backbone"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Dec 24, 2019): https://6900dkn8.network.local/pg6pd9jx/ip=203.0.113.198", "mitigationString": "", "rule": "Historical Multicategory Blocklist", "timestamp": "2019-12-24T09:53:13.546Z"}], "riskString": "1/54", "riskSummary": "1 of 54 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2019-12-24T09:54:02.935Z", "lastSeen": "2021-06-19T19:39:25.532Z"}} +{"entity": {"id": "ip:192.0.2.179", "name": "192.0.2.179", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.179", "location": {"asn": "AS9829", "cidr": {"id": "ip:59.99.200.0/21", "name": "59.99.200.0/21", "type": "IpAddress"}, "location": {"city": "Bangalore", "continent": "Asia", "country": "India"}, "organization": "National Internet Backbone"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Mar 3, 2020): https://f0go.network.local/c1c3m9rsl/ip=192.0.2.179", "mitigationString": "", "rule": "Historical Multicategory Blocklist", "timestamp": "2020-03-03T08:08:07.521Z"}, {"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between Apr 7, 2020, and Apr 8, 2020.", "mitigationString": "", "rule": "Historically Reported in Threat List", "timestamp": "2021-06-21T19:53:19.897Z"}, {"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "High Risk activity in CIDR Block.", "mitigationString": "", "rule": "Recorded Future Predictive Risk Model", "timestamp": "2021-06-21T19:53:19.906Z"}], "riskString": "3/54", "riskSummary": "3 of 54 Risk Rules currently observed.", "rules": 3, "score": 15}, "timestamps": {"firstSeen": "2020-03-03T08:10:28.489Z", "lastSeen": "2021-06-19T19:39:11.694Z"}} +{"entity": {"id": "ip:192.0.2.245", "name": "192.0.2.245", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.245", "location": {"asn": "AS45899", "cidr": {"id": "ip:113.170.96.0/20", "name": "113.170.96.0/20", "type": "IpAddress"}, "location": {"city": "Long Phu", "continent": "Asia", "country": "Vietnam"}, "organization": "VNPT Corp"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between May 25, 2021, and May 25, 2021.", "mitigationString": "", "rule": "Historically Reported in Threat List", "timestamp": "2021-06-19T19:50:20.162Z"}], "riskString": "1/54", "riskSummary": "1 of 54 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-19T19:38:57.372Z", "lastSeen": "2021-06-19T19:38:57.372Z"}} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log-expected.json new file mode 100644 index 00000000000..ed121c0a418 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log-expected.json @@ -0,0 +1,414 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 31287, + "threatintel.indicator.as.organization.name": "IPACCT CABLE Ltd", + "threatintel.indicator.first_seen": "2021-04-18T00:11:48.512Z", + "threatintel.indicator.geo.city_name": "Radnevo", + "threatintel.indicator.geo.continent_name": "Europe", + "threatintel.indicator.geo.country_name": "Bulgaria", + "threatintel.indicator.last_seen": "2021-06-19T19:40:32.897Z", + "threatintel.indicator.type": "ipv6-addr", + "threatintel.recordedfuture.entity.id": "ip:2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.entity.name": "2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.ip_range": "2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 763, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 197207, + "threatintel.indicator.as.organization.name": "Mobile Communication Company of Iran PLC", + "threatintel.indicator.first_seen": "2021-06-19T17:55:58.019Z", + "threatintel.indicator.geo.city_name": null, + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "Iran", + "threatintel.indicator.ip": "2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "threatintel.indicator.last_seen": "2021-06-19T19:40:32.839Z", + "threatintel.indicator.type": "ipv6-addr", + "threatintel.recordedfuture.entity.id": "ip:2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "threatintel.recordedfuture.ip_range": "2001:db8:f800:5c3f:c9f8:fbf8:d537:9071/128", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.55", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1531, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.organization.name": null, + "threatintel.indicator.first_seen": "2021-06-19T19:40:30.596Z", + "threatintel.indicator.geo.city_name": null, + "threatintel.indicator.geo.continent_name": null, + "threatintel.indicator.geo.country_name": null, + "threatintel.indicator.ip": "203.0.113.55", + "threatintel.indicator.last_seen": "2021-06-19T19:40:30.596Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.55", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.55", + "threatintel.recordedfuture.ip_range": "203.0.113.55/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.108", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2161, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 17622, + "threatintel.indicator.as.organization.name": "China Unicom Guangzhou network", + "threatintel.indicator.first_seen": "2021-06-19T19:40:20.534Z", + "threatintel.indicator.geo.city_name": "Guangzhou", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "China", + "threatintel.indicator.ip": "203.0.113.108", + "threatintel.indicator.last_seen": "2021-06-19T19:40:20.534Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.108", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.108", + "threatintel.recordedfuture.ip_range": "203.0.113.108/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.139", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2851, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 7713, + "threatintel.indicator.as.organization.name": "PT Telekomunikasi Indonesia", + "threatintel.indicator.first_seen": "2016-06-23T07:39:06.418Z", + "threatintel.indicator.geo.city_name": null, + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "Indonesia", + "threatintel.indicator.ip": "203.0.113.139", + "threatintel.indicator.last_seen": "2021-06-19T19:40:03.882Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.139", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.139", + "threatintel.recordedfuture.ip_range": "203.0.113.139/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3532, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 17622, + "threatintel.indicator.as.organization.name": "China Unicom Guangzhou network", + "threatintel.indicator.first_seen": "2021-06-19T19:40:02.557Z", + "threatintel.indicator.geo.city_name": "Guangzhou", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "China", + "threatintel.indicator.ip": "2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "threatintel.indicator.last_seen": "2021-06-19T19:40:02.557Z", + "threatintel.indicator.type": "ipv6-addr", + "threatintel.recordedfuture.entity.id": "ip:2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "threatintel.recordedfuture.ip_range": "2001:db8:bf58:c5c3:7a06:5267:82e0:621a/128", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.147", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4295, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 4837, + "threatintel.indicator.as.organization.name": "CHINA UNICOM China169 Backbone", + "threatintel.indicator.first_seen": "2017-12-20T02:21:07.734Z", + "threatintel.indicator.geo.city_name": "Zhengzhou", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "China", + "threatintel.indicator.ip": "192.0.2.147", + "threatintel.indicator.last_seen": "2021-06-19T19:39:43.160Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:192.0.2.147", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.147", + "threatintel.recordedfuture.ip_range": "192.0.2.147/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.198", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4972, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 9829, + "threatintel.indicator.as.organization.name": "National Internet Backbone", + "threatintel.indicator.first_seen": "2019-12-24T09:54:02.935Z", + "threatintel.indicator.geo.city_name": "Palakkad", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "India", + "threatintel.indicator.ip": "203.0.113.198", + "threatintel.indicator.last_seen": "2021-06-19T19:39:25.532Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.198", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.198", + "threatintel.recordedfuture.ip_range": "203.0.113.198/32", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Dec 24, 2019): https://6900dkn8.network.local/pg6pd9jx/ip=203.0.113.198", + "mitigationString": "", + "rule": "Historical Multicategory Blocklist", + "timestamp": "2019-12-24T09:53:13.546Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/54", + "threatintel.recordedfuture.risk.riskSummary": "1 of 54 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.179", + "event.risk_score": 15.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 5970, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 9829, + "threatintel.indicator.as.organization.name": "National Internet Backbone", + "threatintel.indicator.first_seen": "2020-03-03T08:10:28.489Z", + "threatintel.indicator.geo.city_name": "Bangalore", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "India", + "threatintel.indicator.ip": "192.0.2.179", + "threatintel.indicator.last_seen": "2021-06-19T19:39:11.694Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:192.0.2.179", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.179", + "threatintel.recordedfuture.ip_range": "192.0.2.179/32", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Mar 3, 2020): https://f0go.network.local/c1c3m9rsl/ip=192.0.2.179", + "mitigationString": "", + "rule": "Historical Multicategory Blocklist", + "timestamp": "2020-03-03T08:08:07.521Z" + }, + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "High Risk activity in CIDR Block.", + "mitigationString": "", + "rule": "Recorded Future Predictive Risk Model", + "timestamp": "2021-06-21T19:53:19.906Z" + }, + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between Apr 7, 2020, and Apr 8, 2020.", + "mitigationString": "", + "rule": "Historically Reported in Threat List", + "timestamp": "2021-06-21T19:53:19.897Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "3/54", + "threatintel.recordedfuture.risk.riskSummary": "3 of 54 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 3, + "threatintel.recordedfuture.risk.score": 15 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.245", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 7483, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 45899, + "threatintel.indicator.as.organization.name": "VNPT Corp", + "threatintel.indicator.first_seen": "2021-06-19T19:38:57.372Z", + "threatintel.indicator.geo.city_name": "Long Phu", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "Vietnam", + "threatintel.indicator.ip": "192.0.2.245", + "threatintel.indicator.last_seen": "2021-06-19T19:38:57.372Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:192.0.2.245", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.245", + "threatintel.recordedfuture.ip_range": "192.0.2.245/32", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between May 25, 2021, and May 25, 2021.", + "mitigationString": "", + "rule": "Historically Reported in Threat List", + "timestamp": "2021-06-19T19:50:20.162Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/54", + "threatintel.recordedfuture.risk.riskSummary": "1 of 54 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log new file mode 100644 index 00000000000..2016b319f60 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "url:https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", "name": "https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: Qakbot, qbot, zip. Most recent link (Apr 15, 2021): https://jc6.network.local/hnwdzm1k3?url=cdb14", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-04-15T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj"} +{"entity": {"id": "url:https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", "name": "https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: Bitdefender. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on Feb 14, 2021.", "mitigationString": "", "rule": "Historically Detected Phishing Techniques", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61"} +{"entity": {"id": "url:https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", "name": "https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: gafgyt, elf. Most recent link (May 15, 2021): https://bm1.network.local/82p3t?url=12b56", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-05-15T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc"} +{"entity": {"id": "url:https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", "name": "https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://ois8bq4.network.local/4lf?url=3f7c2", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0"} +{"entity": {"id": "url:http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", "name": "http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: @Sh1ttyKids. Most recent tweet: https://8g7zl.network.local/v5hcdo?url=efed5", "mitigationString": "", "rule": "Historically Reported as a Defanged URL", "timestamp": "2020-06-24T12:01:33.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii"} +{"entity": {"id": "url:http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", "name": "http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: . Most recent link (Feb 14, 2021): https://w6l3t5s.network.local/dr2rg5o?url=7b29d", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont"} +{"entity": {"id": "url:http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", "name": "http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Nov 16, 2020): https://0c39b.network.local/tcxah?url=0d1b9", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2020-11-16T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513"} +{"entity": {"id": "url:https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", "name": "https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (May 15, 2021): https://i1-yo.network.local/8-w8hhq2p?url=f7769", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-05-15T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r"} +{"entity": {"id": "url:http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", "name": "http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Feb 14, 2021): https://ysn.network.local/5p-09h7b?url=ff1c3", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2"} +{"entity": {"id": "url:http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", "name": "http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://j-8uc.network.local/2odpjhia?url=1a2ba", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1"} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log-expected.json new file mode 100644 index 00000000000..b341365e428 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log-expected.json @@ -0,0 +1,442 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "d6s.example.net", + "threatintel.indicator.url.original": "https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "threatintel.indicator.url.path": "/nzy/vvc68ke", + "threatintel.indicator.url.query": "p5uxwn=1bj", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: Qakbot, qbot, zip. Most recent link (Apr 15, 2021): https://jc6.network.local/hnwdzm1k3?url=cdb14", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-04-15T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 874, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "ga7v9u.example.org", + "threatintel.indicator.url.original": "https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "threatintel.indicator.url.path": "/bnqv8e2v8/qb49", + "threatintel.indicator.url.query": "7kq=iw61", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: Bitdefender. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on Feb 14, 2021.", + "mitigationString": "", + "rule": "Historically Detected Phishing Techniques", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1760, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "cdmw.example.net", + "threatintel.indicator.url.original": "https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "threatintel.indicator.url.path": "/c20fwa/wwn", + "threatintel.indicator.url.query": "dlz53=z6ovc", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: gafgyt, elf. Most recent link (May 15, 2021): https://bm1.network.local/82p3t?url=12b56", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-05-15T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2627, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "4mne.example.local", + "threatintel.indicator.url.original": "https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "threatintel.indicator.url.path": "/ns2rk8f/wngtk2xz", + "threatintel.indicator.url.query": "vceuk7wl6=3p0", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://ois8bq4.network.local/4lf?url=3f7c2", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3524, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "z198hloc8.example.com", + "threatintel.indicator.url.original": "http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "threatintel.indicator.url.path": "/f8ih39/f6kou", + "threatintel.indicator.url.query": "f6-u3=uwhii", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: @Sh1ttyKids. Most recent tweet: https://8g7zl.network.local/v5hcdo?url=efed5", + "mitigationString": "", + "rule": "Historically Reported as a Defanged URL", + "timestamp": "2020-06-24T12:01:33.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4377, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "y484j-fb6.example.local", + "threatintel.indicator.url.original": "http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "threatintel.indicator.url.path": "/b97s24xf/prz", + "threatintel.indicator.url.query": "sg-x1do=4myont", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: . Most recent link (Feb 14, 2021): https://w6l3t5s.network.local/dr2rg5o?url=7b29d", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 5272, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "sp2xyqq82.example.local", + "threatintel.indicator.url.original": "http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "threatintel.indicator.url.path": "/zxvm093/kat1rcz", + "threatintel.indicator.url.query": "vaev0aeod=rc0513", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Nov 16, 2020): https://0c39b.network.local/tcxah?url=0d1b9", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2020-11-16T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 6187, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "zh4o7xc.example.com", + "threatintel.indicator.url.original": "https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "threatintel.indicator.url.path": "/-yiq/vg2whtxif", + "threatintel.indicator.url.query": "cb0-knk=s6poib5r", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (May 15, 2021): https://i1-yo.network.local/8-w8hhq2p?url=f7769", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-05-15T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 7094, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "fiivf4s.example.org", + "threatintel.indicator.url.original": "http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "threatintel.indicator.url.path": "/8u2qi/86vfcfq7m", + "threatintel.indicator.url.query": "pfb2ensc0=h7imk8io2", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Feb 14, 2021): https://ysn.network.local/5p-09h7b?url=ff1c3", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 8007, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "abav9v.example.org", + "threatintel.indicator.url.original": "http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "threatintel.indicator.url.path": "/gj93q/7fs7", + "threatintel.indicator.url.query": "kcq7=pjaj1", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://j-8uc.network.local/2odpjhia?url=1a2ba", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/threatintel.yml.disabled b/x-pack/filebeat/modules.d/threatintel.yml.disabled index 255cd30aa40..e150fe8835a 100644 --- a/x-pack/filebeat/modules.d/threatintel.yml.disabled +++ b/x-pack/filebeat/modules.d/threatintel.yml.disabled @@ -140,3 +140,35 @@ # # var.ssl_certificate: path/to/server_ssl_cert.pem # var.ssl_key: path/to/ssl_key.pem + + recordedfuture: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The interval to poll the API for updates + var.interval: 5m + + # How far back in time to start fetching intelligence when run for the + # first time. Value must be in hours. Default: 168h (1 week). + var.first_interval: 168h + + # The URL used for Threat Intel API calls. + # Must include the `limit` parameter and at least `entity` and `timestamps` fields. + # See the Connect API Explorer for a list of possible parameters. + # + # For `ip` entities: + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `domain` entities: + # var.url: "https://api.recordedfuture.com/v2/domain/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `hash` entities: + # var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + + # For `url` entities: + # var.url: "https://api.recordedfuture.com/v2/url/search?limit=200&fields=entity,timestamps,risk&metadata=false" + + # Set your API Token. + var.api_token: "" From 785cc26df6e52fcf3604f98a1f42a1de4fe92131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patrik=20Pihlstr=C3=B6m?= Date: Mon, 28 Jun 2021 14:37:43 +0200 Subject: [PATCH 02/18] Update shared-deduplication.asciidoc (#26492) --- libbeat/docs/shared-deduplication.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/docs/shared-deduplication.asciidoc b/libbeat/docs/shared-deduplication.asciidoc index eec45caaf97..e0d1f06e1aa 100644 --- a/libbeat/docs/shared-deduplication.asciidoc +++ b/libbeat/docs/shared-deduplication.asciidoc @@ -114,7 +114,7 @@ input { beats { port => 5044 } -}} +} output { if [@metadata][_id] { From 1c9a488fbd5a2cf32c0d15dd8285179a3f425861 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Mon, 28 Jun 2021 21:57:04 +0800 Subject: [PATCH 03/18] Add log_group_name_prefix config option for aws-cloudwatch input (#26187) --- CHANGELOG.next.asciidoc | 1 + .../filebeat.inputs.reference.xpack.yml.tmpl | 57 +++++++++++++++++ .../docs/inputs/input-aws-cloudwatch.asciidoc | 11 +++- x-pack/filebeat/filebeat.reference.yml | 57 +++++++++++++++++ x-pack/filebeat/input/awscloudwatch/config.go | 15 +++-- x-pack/filebeat/input/awscloudwatch/input.go | 62 ++++++++++++++++--- 6 files changed, 187 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5e09e0908ab..2011f13bec5 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -818,6 +818,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686] - Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774] - Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] +- Add log_group_name_prefix config into aws-cloudwatch input. {pull}26187[26187] - Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168] - http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764] - Make `filestream` input GA. {pull}26127[26127] diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl index 8b2b53fa8c7..13cb37c0f9c 100644 --- a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl +++ b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl @@ -76,3 +76,60 @@ # List of S3 object metadata keys to include in events. #include_s3_metadata: [] + +#------------------------------ AWS CloudWatch input -------------------------------- +# Beta: Config options for AWS CloudWatch input +#- type: aws-cloudwatch + #enabled: false + + # AWS Credentials + # If access_key_id and secret_access_key are configured, then use them to make api calls. + # If not, aws-cloudwatch input will load default AWS config or load with given profile name. + #access_key_id: '${AWS_ACCESS_KEY_ID:""}' + #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + #session_token: '${AWS_SESSION_TOKEN:"”}' + #credential_profile_name: test-aws-s3-input + + # ARN of the log group to collect logs from + #log_group_arn: "arn:aws:logs:us-east-1:428152502467:log-group:test:*" + + # Name of the log group to collect logs from. + # Note: region_name is required when log_group_name is given. + #log_group_name: test + + # The prefix for a group of log group names. + # Note: `region_name` is required when `log_group_name_prefix` is given. + # `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + #log_group_name_prefix: /aws/ + + # Region that the specified log group or log group prefix belongs to. + #region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from. + #log_streams: + # - log_stream_name + + # A string to filter the results to include only log events from log streams + # that have names starting with this prefix. + #log_stream_prefix: test + + # `start_position` allows user to specify if this input should read log files + # from the `beginning` or from the `end`. + # `beginning`: reads from the beginning of the log group (default). + # `end`: read only new messages from current time minus `scan_frequency` going forward. + #start_position: beginning + + # This config parameter sets how often Filebeat checks for new log events from the + # specified log group. Default `scan_frequency` is 1 minute, which means Filebeat + # will sleep for 1 minute before querying for new logs again. + #scan_frequency: 1m + + # The maximum duration of AWS API can take. If it exceeds the timeout, AWS API + # will be interrupted. + # The default AWS API timeout for a message is 120 seconds. + # The minimum is 0 seconds. + #api_timeout: 120s + + # This is used to sleep between AWS `FilterLogEvents` API calls inside the same + # collection period. + #api_sleep: 200ms diff --git a/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc index d39ec550868..e04eaa8b9e2 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc @@ -45,12 +45,18 @@ ARN of the log group to collect logs from. [float] ==== `log_group_name` -Name of the log group to collect logs from. Note: region_name is required when +Name of the log group to collect logs from. Note: `region_name` is required when log_group_name is given. +[float] +==== `log_group_name_prefix` +The prefix for a group of log group names. Note: `region_name` is required when +log_group_name_prefix is given. `log_group_name` and `log_group_name_prefix` +cannot be given at the same time. + [float] ==== `region_name` -Region that the specified log group belongs to. +Region that the specified log group or log group prefix belongs to. [float] ==== `log_streams` @@ -109,6 +115,7 @@ Please see <> for more details. === AWS Permissions Specific AWS permissions are required for IAM user to access aws-cloudwatch: ---- +cloudwatchlogs:DescribeLogGroups logs:FilterLogEvents ---- diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index ecbcc538f16..9a797d337fc 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -3127,6 +3127,63 @@ filebeat.inputs: # List of S3 object metadata keys to include in events. #include_s3_metadata: [] +#------------------------------ AWS CloudWatch input -------------------------------- +# Beta: Config options for AWS CloudWatch input +#- type: aws-cloudwatch + #enabled: false + + # AWS Credentials + # If access_key_id and secret_access_key are configured, then use them to make api calls. + # If not, aws-cloudwatch input will load default AWS config or load with given profile name. + #access_key_id: '${AWS_ACCESS_KEY_ID:""}' + #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + #session_token: '${AWS_SESSION_TOKEN:"”}' + #credential_profile_name: test-aws-s3-input + + # ARN of the log group to collect logs from + #log_group_arn: "arn:aws:logs:us-east-1:428152502467:log-group:test:*" + + # Name of the log group to collect logs from. + # Note: region_name is required when log_group_name is given. + #log_group_name: test + + # The prefix for a group of log group names. + # Note: `region_name` is required when `log_group_name_prefix` is given. + # `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + #log_group_name_prefix: /aws/ + + # Region that the specified log group or log group prefix belongs to. + #region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from. + #log_streams: + # - log_stream_name + + # A string to filter the results to include only log events from log streams + # that have names starting with this prefix. + #log_stream_prefix: test + + # `start_position` allows user to specify if this input should read log files + # from the `beginning` or from the `end`. + # `beginning`: reads from the beginning of the log group (default). + # `end`: read only new messages from current time minus `scan_frequency` going forward. + #start_position: beginning + + # This config parameter sets how often Filebeat checks for new log events from the + # specified log group. Default `scan_frequency` is 1 minute, which means Filebeat + # will sleep for 1 minute before querying for new logs again. + #scan_frequency: 1m + + # The maximum duration of AWS API can take. If it exceeds the timeout, AWS API + # will be interrupted. + # The default AWS API timeout for a message is 120 seconds. + # The minimum is 0 seconds. + #api_timeout: 120s + + # This is used to sleep between AWS `FilterLogEvents` API calls inside the same + # collection period. + #api_sleep: 200ms + # =========================== Filebeat autodiscover ============================ # Autodiscover allows you to detect changes in the system and spawn new modules diff --git a/x-pack/filebeat/input/awscloudwatch/config.go b/x-pack/filebeat/input/awscloudwatch/config.go index 7f1e4ca3ccd..210e7bb9a31 100644 --- a/x-pack/filebeat/input/awscloudwatch/config.go +++ b/x-pack/filebeat/input/awscloudwatch/config.go @@ -16,6 +16,7 @@ type config struct { harvester.ForwarderConfig `config:",inline"` LogGroupARN string `config:"log_group_arn"` LogGroupName string `config:"log_group_name"` + LogGroupNamePrefix string `config:"log_group_name_prefix"` RegionName string `config:"region_name"` LogStreams []string `config:"log_streams"` LogStreamPrefix string `config:"log_stream_prefix"` @@ -44,13 +45,17 @@ func (c *config) Validate() error { "either 'beginning' or 'end'") } - if c.LogGroupARN == "" && c.LogGroupName == "" { - return errors.New("log_group_arn and log_group_name config parameter" + - "cannot be both empty") + if c.LogGroupARN == "" && c.LogGroupName == "" && c.LogGroupNamePrefix == "" { + return errors.New("log_group_arn, log_group_name and log_group_name_prefix config parameter" + + "cannot all be empty") } - if c.LogGroupName != "" && c.RegionName == "" { - return errors.New("region_name is required when log_group_name " + + if c.LogGroupName != "" && c.LogGroupNamePrefix != "" { + return errors.New("log_group_name and log_group_name_prefix cannot be given at the same time") + } + + if (c.LogGroupName != "" || c.LogGroupNamePrefix != "") && c.RegionName == "" { + return errors.New("region_name is required when log_group_name or log_group_name_prefix " + "config parameter is given") } return nil diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go index 2f3563a369e..7eb14147d31 100644 --- a/x-pack/filebeat/input/awscloudwatch/input.go +++ b/x-pack/filebeat/input/awscloudwatch/input.go @@ -131,15 +131,33 @@ func NewInput(cfg *common.Config, connector channel.Connector, context input.Con // Run runs the input func (in *awsCloudWatchInput) Run() { - in.workerOnce.Do(func() { - in.workerWg.Add(1) - go func() { - in.logger.Infof("aws-cloudwatch input worker for log group: '%v' has started", in.config.LogGroupName) - defer in.logger.Infof("aws-cloudwatch input worker for log group '%v' has stopped.", in.config.LogGroupName) - defer in.workerWg.Done() - in.run() - }() - }) + cwConfig := awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "cloudwatchlogs", in.config.RegionName, in.awsConfig) + svc := cloudwatchlogs.New(cwConfig) + + var logGroupNames []string + var err error + if in.config.LogGroupNamePrefix != "" { + logGroupNames, err = in.getLogGroupNames(svc) + if err != nil { + in.logger.Error("getLogGroupNames failed: ", err) + return + } + } else { + logGroupNames = []string{in.config.LogGroupName} + } + + for _, logGroup := range logGroupNames { + in.config.LogGroupName = logGroup + in.workerOnce.Do(func() { + in.workerWg.Add(1) + go func() { + in.logger.Infof("aws-cloudwatch input worker for log group: '%v' has started", in.config.LogGroupName) + defer in.logger.Infof("aws-cloudwatch input worker for log group '%v' has stopped.", in.config.LogGroupName) + defer in.workerWg.Done() + in.run() + }() + }) + } } func (in *awsCloudWatchInput) run() { @@ -176,6 +194,32 @@ func parseARN(logGroupARN string) (string, string, error) { return "", "", errors.Errorf("cannot get log group name from log group ARN: %s", logGroupARN) } +// getLogGroupNames uses DescribeLogGroups API to retrieve all log group names +func (in *awsCloudWatchInput) getLogGroupNames(svc cloudwatchlogsiface.ClientAPI) ([]string, error) { + // construct DescribeLogGroupsInput + filterLogEventsInput := &cloudwatchlogs.DescribeLogGroupsInput{ + LogGroupNamePrefix: awssdk.String(in.config.LogGroupNamePrefix), + } + + // make API request + req := svc.DescribeLogGroupsRequest(filterLogEventsInput) + p := cloudwatchlogs.NewDescribeLogGroupsPaginator(req) + var logGroupNames []string + for p.Next(context.TODO()) { + page := p.CurrentPage() + in.logger.Debugf("Collecting #%v log group names", len(page.LogGroups)) + for _, lg := range page.LogGroups { + logGroupNames = append(logGroupNames, *lg.LogGroupName) + } + } + + if err := p.Err(); err != nil { + in.logger.Error("failed DescribeLogGroupsRequest: ", err) + return logGroupNames, err + } + return logGroupNames, nil +} + // getLogEventsFromCloudWatch uses FilterLogEvents API to collect logs from CloudWatch func (in *awsCloudWatchInput) getLogEventsFromCloudWatch(svc cloudwatchlogsiface.ClientAPI) error { ctx, cancelFn := context.WithTimeout(in.inputCtx, in.config.APITimeout) From a3b447ac4e0930cf0525f257a2e63ef1de4bc7b6 Mon Sep 17 00:00:00 2001 From: Fae Charlton Date: Mon, 28 Jun 2021 11:05:55 -0400 Subject: [PATCH 04/18] [libbeat] Fix encoding and file offset issues in the disk queue (#26484) --- CHANGELOG.next.asciidoc | 1 + .../queue/diskqueue/core_loop_test.go | 6 +- .../publisher/queue/diskqueue/reader_loop.go | 26 +++---- libbeat/publisher/queue/diskqueue/segments.go | 23 +++++-- .../publisher/queue/diskqueue/serialize.go | 20 ++++-- .../queue/diskqueue/serialize_test.go | 69 +++++++++++++++++++ .../publisher/queue/diskqueue/writer_loop.go | 23 ++++--- 7 files changed, 132 insertions(+), 36 deletions(-) create mode 100644 libbeat/publisher/queue/diskqueue/serialize_test.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2011f13bec5..48f5b600b76 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -236,6 +236,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143] - Omit full index template from errors that occur while loading the template. {pull}25743[25743] - In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively. +- Fix encoding errors when using the disk queue on nested data with multi-byte characters {pull}26484[26484] *Auditbeat* diff --git a/libbeat/publisher/queue/diskqueue/core_loop_test.go b/libbeat/publisher/queue/diskqueue/core_loop_test.go index 3bf53e9b8a5..7c0d426ea61 100644 --- a/libbeat/publisher/queue/diskqueue/core_loop_test.go +++ b/libbeat/publisher/queue/diskqueue/core_loop_test.go @@ -235,7 +235,7 @@ func TestHandleWriterLoopResponse(t *testing.T) { // Write to one segment (no segments should be moved to reading list) dq.handleWriterLoopResponse(writerLoopResponse{ - segments: []writerLoopResponseSegment{ + segments: []writerLoopSegmentResponse{ {bytesWritten: 100}, }, }) @@ -250,7 +250,7 @@ func TestHandleWriterLoopResponse(t *testing.T) { // Write to two segments (the first one should be moved to reading list) dq.handleWriterLoopResponse(writerLoopResponse{ - segments: []writerLoopResponseSegment{ + segments: []writerLoopSegmentResponse{ {bytesWritten: 100}, {bytesWritten: 100}, }, @@ -270,7 +270,7 @@ func TestHandleWriterLoopResponse(t *testing.T) { // Write to three segments (the first two should be moved to reading list) dq.handleWriterLoopResponse(writerLoopResponse{ - segments: []writerLoopResponseSegment{ + segments: []writerLoopSegmentResponse{ {bytesWritten: 100}, {bytesWritten: 100}, {bytesWritten: 500}, diff --git a/libbeat/publisher/queue/diskqueue/reader_loop.go b/libbeat/publisher/queue/diskqueue/reader_loop.go index a31cc5b8a77..d6bb49494e0 100644 --- a/libbeat/publisher/queue/diskqueue/reader_loop.go +++ b/libbeat/publisher/queue/diskqueue/reader_loop.go @@ -20,6 +20,7 @@ package diskqueue import ( "encoding/binary" "fmt" + "io" "os" ) @@ -100,13 +101,12 @@ func (rl *readerLoop) processRequest(request readerLoopRequest) readerLoopRespon // Open the file and seek to the starting position. handle, err := request.segment.getReader(rl.settings) + rl.decoder.useJSON = request.segment.shouldUseJSON() if err != nil { return readerLoopResponse{err: err} } defer handle.Close() - // getReader positions us at the start of the data region, so we use - // a relative seek to advance to the request position. - _, err = handle.Seek(int64(request.startPosition), os.SEEK_CUR) + _, err = handle.Seek(int64(request.startPosition), io.SeekStart) if err != nil { return readerLoopResponse{err: err} } @@ -179,7 +179,7 @@ func (rl *readerLoop) nextFrame( // Ensure we are allowed to read the frame header. if maxLength < frameHeaderSize { return nil, fmt.Errorf( - "Can't read next frame: remaining length %d is too low", maxLength) + "can't read next frame: remaining length %d is too low", maxLength) } // Wrap the handle to retry non-fatal errors and always return the full // requested data length if possible. @@ -187,20 +187,20 @@ func (rl *readerLoop) nextFrame( var frameLength uint32 err := binary.Read(reader, binary.LittleEndian, &frameLength) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame header: %w", err) + return nil, fmt.Errorf("couldn't read data frame header: %w", err) } // If the frame extends past the area we were told to read, return an error. // This should never happen unless the segment file is corrupted. if maxLength < uint64(frameLength) { return nil, fmt.Errorf( - "Can't read next frame: frame size is %d but remaining data is only %d", + "can't read next frame: frame size is %d but remaining data is only %d", frameLength, maxLength) } if frameLength <= frameMetadataSize { // Valid enqueued data must have positive length return nil, fmt.Errorf( - "Data frame with no data (length %d)", frameLength) + "data frame with no data (length %d)", frameLength) } // Read the actual frame data @@ -208,29 +208,29 @@ func (rl *readerLoop) nextFrame( bytes := rl.decoder.Buffer(int(dataLength)) _, err = reader.Read(bytes) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame content: %w", err) + return nil, fmt.Errorf("couldn't read data frame content: %w", err) } // Read the footer (checksum + duplicate length) var checksum uint32 err = binary.Read(reader, binary.LittleEndian, &checksum) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame checksum: %w", err) + return nil, fmt.Errorf("couldn't read data frame checksum: %w", err) } expected := computeChecksum(bytes) if checksum != expected { return nil, fmt.Errorf( - "Data frame checksum mismatch (%x != %x)", checksum, expected) + "data frame checksum mismatch (%x != %x)", checksum, expected) } var duplicateLength uint32 err = binary.Read(reader, binary.LittleEndian, &duplicateLength) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame footer: %w", err) + return nil, fmt.Errorf("couldn't read data frame footer: %w", err) } if duplicateLength != frameLength { return nil, fmt.Errorf( - "Inconsistent data frame length (%d vs %d)", + "inconsistent data frame length (%d vs %d)", frameLength, duplicateLength) } @@ -242,7 +242,7 @@ func (rl *readerLoop) nextFrame( // TODO: Rather than pass this error back to the read request, which // discards the rest of the segment, we should just log the error and // advance to the next frame, which is likely still valid. - return nil, fmt.Errorf("Couldn't decode data frame: %w", err) + return nil, fmt.Errorf("couldn't decode data frame: %w", err) } frame := &readFrame{ diff --git a/libbeat/publisher/queue/diskqueue/segments.go b/libbeat/publisher/queue/diskqueue/segments.go index f666a1941c8..450139b2d04 100644 --- a/libbeat/publisher/queue/diskqueue/segments.go +++ b/libbeat/publisher/queue/diskqueue/segments.go @@ -91,7 +91,10 @@ type queueSegment struct { // If this segment was loaded from a previous session, schemaVersion // points to the file schema version that was read from its header. // This is only used by queueSegment.headerSize(), which is used in - // maybeReadPending to calculate the position of the first data frame. + // maybeReadPending to calculate the position of the first data frame, + // and by queueSegment.shouldUseJSON(), which is used in the reader + // loop to detect old segments that used JSON encoding instead of + // the current CBOR. schemaVersion *uint32 // The number of bytes occupied by this segment on-disk, as of the most @@ -198,6 +201,14 @@ func (segment *queueSegment) headerSize() uint64 { return segmentHeaderSize } +// The initial release of the disk queue used JSON to encode events +// on disk. Since then, we have switched to CBOR to address issues +// with encoding multi-byte characters, and for lower encoding +// overhead. +func (segment *queueSegment) shouldUseJSON() bool { + return segment.schemaVersion != nil && *segment.schemaVersion == 0 +} + // Should only be called from the reader loop. If successful, returns an open // file handle positioned at the beginning of the segment's data region. func (segment *queueSegment) getReader( @@ -207,14 +218,14 @@ func (segment *queueSegment) getReader( file, err := os.Open(path) if err != nil { return nil, fmt.Errorf( - "Couldn't open segment %d: %w", segment.id, err) + "couldn't open segment %d: %w", segment.id, err) } // We don't need the header contents here, we just want to advance past the // header region, so discard the return value. _, err = readSegmentHeader(file) if err != nil { file.Close() - return nil, fmt.Errorf("Couldn't read segment header: %w", err) + return nil, fmt.Errorf("couldn't read segment header: %w", err) } return file, nil @@ -231,7 +242,7 @@ func (segment *queueSegment) getWriter( } err = writeSegmentHeader(file, 0) if err != nil { - return nil, fmt.Errorf("Couldn't write segment header: %w", err) + return nil, fmt.Errorf("couldn't write segment header: %w", err) } return file, nil @@ -306,7 +317,7 @@ func readSegmentHeaderWithFrameCount(path string) (*segmentHeader, error) { // the current frame to make sure the trailing length matches before // advancing to the next frame (otherwise we might accept an impossible // length). - _, err = file.Seek(int64(frameLength-8), os.SEEK_CUR) + _, err = file.Seek(int64(frameLength-8), io.SeekCurrent) if err != nil { break } @@ -341,7 +352,7 @@ func readSegmentHeader(in io.Reader) (*segmentHeader, error) { return nil, err } if header.version > currentSegmentVersion { - return nil, fmt.Errorf("Unrecognized schema version %d", header.version) + return nil, fmt.Errorf("unrecognized schema version %d", header.version) } if header.version >= 1 { err = binary.Read(in, binary.LittleEndian, &header.frameCount) diff --git a/libbeat/publisher/queue/diskqueue/serialize.go b/libbeat/publisher/queue/diskqueue/serialize.go index 456f180665b..fc0c24de673 100644 --- a/libbeat/publisher/queue/diskqueue/serialize.go +++ b/libbeat/publisher/queue/diskqueue/serialize.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/outputs/codec" "github.com/elastic/beats/v7/libbeat/publisher" + "github.com/elastic/go-structform/cborl" "github.com/elastic/go-structform/gotype" "github.com/elastic/go-structform/json" ) @@ -40,7 +41,13 @@ type eventEncoder struct { type eventDecoder struct { buf []byte - parser *json.Parser + jsonParser *json.Parser + cborlParser *cborl.Parser + + // Current serialization all uses CBOR. Set this flag when decoding + // from old (schema 0) segment files generated by the disk queue beta. + useJSON bool + unfolder *gotype.Unfolder } @@ -60,7 +67,7 @@ func newEventEncoder() *eventEncoder { func (e *eventEncoder) reset() { e.folder = nil - visitor := json.NewVisitor(&e.buf) + visitor := cborl.NewVisitor(&e.buf) // This can't return an error: NewIterator is deterministic based on its // input, and doesn't return an error when called with valid options. In // this case the options are hard-coded to fixed values, so they are @@ -109,7 +116,8 @@ func (d *eventDecoder) reset() { unfolder, _ := gotype.NewUnfolder(nil) d.unfolder = unfolder - d.parser = json.NewParser(unfolder) + d.jsonParser = json.NewParser(unfolder) + d.cborlParser = cborl.NewParser(unfolder) } // Buffer prepares the read buffer to hold the next event of n bytes. @@ -131,7 +139,11 @@ func (d *eventDecoder) Decode() (publisher.Event, error) { d.unfolder.SetTarget(&to) defer d.unfolder.Reset() - err = d.parser.Parse(d.buf) + if d.useJSON { + err = d.jsonParser.Parse(d.buf) + } else { + err = d.cborlParser.Parse(d.buf) + } if err != nil { d.reset() // reset parser just in case diff --git a/libbeat/publisher/queue/diskqueue/serialize_test.go b/libbeat/publisher/queue/diskqueue/serialize_test.go new file mode 100644 index 00000000000..888a42eca88 --- /dev/null +++ b/libbeat/publisher/queue/diskqueue/serialize_test.go @@ -0,0 +1,69 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package diskqueue + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/publisher" +) + +// A test to make sure serialization works correctly on multi-byte characters. +func TestSerialize(t *testing.T) { + testCases := []struct { + name string + value string + }{ + {name: "Ascii only", value: "{\"name\": \"Momotaro\"}"}, + {name: "Multi-byte", value: "{\"name\": \"桃太郎\"}"}, + } + + for _, test := range testCases { + encoder := newEventEncoder() + event := publisher.Event{ + Content: beat.Event{ + Fields: common.MapStr{ + "test_field": test.value, + }, + }, + } + serialized, err := encoder.encode(&event) + if err != nil { + t.Fatalf("[%v] Couldn't encode event: %v", test.name, err) + } + + // Use decoder to decode the serialized bytes. + decoder := newEventDecoder() + buf := decoder.Buffer(len(serialized)) + copy(buf, serialized) + decoded, err := decoder.Decode() + if err != nil { + t.Fatalf("[%v] Couldn't decode serialized data: %v", test.name, err) + } + + decodedValue, err := decoded.Content.Fields.GetValue("test_field") + if err != nil { + t.Fatalf("[%v] Couldn't get field 'test_field': %v", test.name, err) + } + assert.Equal(t, test.value, decodedValue) + } +} diff --git a/libbeat/publisher/queue/diskqueue/writer_loop.go b/libbeat/publisher/queue/diskqueue/writer_loop.go index ce7bed6b2b0..31a03f9d27e 100644 --- a/libbeat/publisher/queue/diskqueue/writer_loop.go +++ b/libbeat/publisher/queue/diskqueue/writer_loop.go @@ -49,9 +49,9 @@ type writerLoopRequest struct { frames []segmentedFrame } -// A writerLoopResponseSegment specifies the number of frames and bytes +// A writerLoopSegmentResponse specifies the number of frames and bytes // written to a single segment as a result of a writerLoopRequest. -type writerLoopResponseSegment struct { +type writerLoopSegmentResponse struct { framesWritten uint32 bytesWritten uint64 } @@ -61,7 +61,7 @@ type writerLoopResponseSegment struct { // segment that appeared in the request, in the same order. If there is // more than one entry, then all but the last segment have been closed. type writerLoopResponse struct { - segments []writerLoopResponseSegment + segments []writerLoopSegmentResponse } type writerLoop struct { @@ -143,7 +143,7 @@ func (wl *writerLoop) processRequest( // responseEntry tracks the number of frames and bytes written to the // current segment. - var curSegment writerLoopResponseSegment + var curSegmentResponse writerLoopSegmentResponse // response var response writerLoopResponse outerLoop: @@ -157,14 +157,14 @@ outerLoop: // Update the header with the frame count (including the ones we // just wrote), try to sync to disk, then close the file. writeSegmentHeader(wl.outputFile, - wl.currentSegment.frameCount+curSegment.framesWritten) + wl.currentSegment.frameCount+curSegmentResponse.framesWritten) wl.outputFile.Sync() wl.outputFile.Close() wl.outputFile = nil // We are done with this segment, add the totals to the response and // reset the current counters. - response.segments = append(response.segments, curSegment) - curSegment = writerLoopResponseSegment{} + response.segments = append(response.segments, curSegmentResponse) + curSegmentResponse = writerLoopSegmentResponse{} } wl.currentSegment = frameRequest.segment file, err := wl.currentSegment.getWriterWithRetry( @@ -173,6 +173,9 @@ outerLoop: // This can only happen if the queue is being closed; abort. break } + // We're creating a new segment file, set the initial bytes written + // to the header size. + curSegmentResponse.bytesWritten = wl.currentSegment.headerSize() wl.outputFile = file } // Make sure our writer points to the current file handle. @@ -208,8 +211,8 @@ outerLoop: // abort while a frame is partially written, we only report up to the // last complete frame. (This almost never matters, but it allows for // more controlled recovery after a bad shutdown.) - curSegment.framesWritten++ - curSegment.bytesWritten += uint64(frameSize) + curSegmentResponse.framesWritten++ + curSegmentResponse.bytesWritten += uint64(frameSize) // Update the ACKs that will be sent at the end of the request. totalACKCount++ @@ -238,7 +241,7 @@ outerLoop: } // Add the final segment to the response and return it. - response.segments = append(response.segments, curSegment) + response.segments = append(response.segments, curSegmentResponse) return response } From 48935ee1c50e7e3f8a148859fbca57090fe7ee5f Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Mon, 28 Jun 2021 18:17:37 +0300 Subject: [PATCH 05/18] Fix master's linting issue (#26517) * Fix master's linting issue Signed-off-by: chrismark * Remove empty lines Signed-off-by: chrismark --- x-pack/osquerybeat/beater/config_plugin_test.go | 2 +- x-pack/osquerybeat/beater/logger_plugin.go | 3 ++- x-pack/osquerybeat/beater/logger_plugin_test.go | 2 +- x-pack/osquerybeat/internal/osqd/osqueryd.go | 3 ++- x-pack/osquerybeat/internal/osqd/osqueryd_windows.go | 3 ++- x-pack/osquerybeat/internal/osqdcli/client.go | 3 ++- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/osquerybeat/beater/config_plugin_test.go b/x-pack/osquerybeat/beater/config_plugin_test.go index 456836a0a59..c8c52371a38 100644 --- a/x-pack/osquerybeat/beater/config_plugin_test.go +++ b/x-pack/osquerybeat/beater/config_plugin_test.go @@ -11,9 +11,9 @@ import ( "path/filepath" "testing" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/google/go-cmp/cmp" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/testutil" ) diff --git a/x-pack/osquerybeat/beater/logger_plugin.go b/x-pack/osquerybeat/beater/logger_plugin.go index 83c510207f5..3160c4d64f3 100644 --- a/x-pack/osquerybeat/beater/logger_plugin.go +++ b/x-pack/osquerybeat/beater/logger_plugin.go @@ -8,8 +8,9 @@ import ( "context" "encoding/json" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/kolide/osquery-go/plugin/logger" + + "github.com/elastic/beats/v7/libbeat/logp" ) type SnapshotResult struct { diff --git a/x-pack/osquerybeat/beater/logger_plugin_test.go b/x-pack/osquerybeat/beater/logger_plugin_test.go index a501d8aebab..d5dd625df67 100644 --- a/x-pack/osquerybeat/beater/logger_plugin_test.go +++ b/x-pack/osquerybeat/beater/logger_plugin_test.go @@ -9,10 +9,10 @@ import ( "encoding/json" "testing" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/google/go-cmp/cmp" "github.com/kolide/osquery-go/plugin/logger" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/testutil" ) diff --git a/x-pack/osquerybeat/internal/osqd/osqueryd.go b/x-pack/osquerybeat/internal/osqd/osqueryd.go index e3de3e00913..f2e5f184bda 100644 --- a/x-pack/osquerybeat/internal/osqd/osqueryd.go +++ b/x-pack/osquerybeat/internal/osqd/osqueryd.go @@ -18,8 +18,9 @@ import ( "time" "github.com/dolmen-go/contextio" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/logp" ) const ( diff --git a/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go b/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go index 22b4f2e820f..758dce74500 100644 --- a/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go +++ b/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go @@ -13,8 +13,9 @@ import ( "github.com/gofrs/uuid" ) + const ( - extensionName = "osquery-extension.exe" + extensionName = "osquery-extension.exe" ) func CreateSocketPath() (string, func(), error) { diff --git a/x-pack/osquerybeat/internal/osqdcli/client.go b/x-pack/osquerybeat/internal/osqdcli/client.go index ae582226573..ceb0fee340f 100644 --- a/x-pack/osquerybeat/internal/osqdcli/client.go +++ b/x-pack/osquerybeat/internal/osqdcli/client.go @@ -14,8 +14,9 @@ import ( "golang.org/x/sync/semaphore" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/kolide/osquery-go" + + "github.com/elastic/beats/v7/libbeat/logp" ) const ( From 2f9ae331de7f213e53ea08ef599a5b24ec02f7ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 28 Jun 2021 17:18:41 +0200 Subject: [PATCH 06/18] Suppress too many errors (#26224) --- journalbeat/input/input.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/journalbeat/input/input.go b/journalbeat/input/input.go index b45b99d1816..f1238d61c81 100644 --- a/journalbeat/input/input.go +++ b/journalbeat/input/input.go @@ -18,12 +18,18 @@ package input import ( + "context" "fmt" + "strings" "sync" + "syscall" + "time" "github.com/elastic/beats/v7/libbeat/processors/add_formatted_index" + "github.com/elastic/go-concert/timed" "github.com/elastic/beats/v7/libbeat/common/acker" + "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/journalbeat/checkpoint" @@ -158,6 +164,10 @@ func (i *Input) publishAll() { go func() { defer wg.Done() + suppressed := atomic.NewBool(false) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for { select { case <-i.done: @@ -168,6 +178,10 @@ func (i *Input) publishAll() { event, err := r.Next() if event == nil { if err != nil { + if i.isErrSuppressed(ctx, err, suppressed) { + i.logger.Debugf("Error message suppressed: EBADMSG") + continue + } i.logger.Errorf("Error while reading event: %v", err) } continue @@ -191,6 +205,26 @@ func (i *Input) publishAll() { } } +// isErrSuppressed checks if the error is due to a corrupt journal. If yes, only the first error message +// is displayed and then it is suppressed for 5 seconds. +func (i *Input) isErrSuppressed(ctx context.Context, err error, suppressed *atomic.Bool) bool { + if strings.Contains(err.Error(), syscall.EBADMSG.Error()) { + if suppressed.Load() { + return true + } + + suppressed.Store(true) + go func(ctx context.Context, suppressed *atomic.Bool) { + if err := timed.Wait(ctx, 5*time.Second); err == nil { + suppressed.Store(false) + } + + }(ctx, suppressed) + } + + return false +} + // Stop stops all readers of the input. func (i *Input) Stop() { for _, r := range i.readers { From 9554d72acaca46a2a86f495de5bff0e8c12d0dde Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 28 Jun 2021 16:50:34 +0100 Subject: [PATCH 07/18] Add inttests for the x-pack/metricbeat on a PR/branches basis (#26526) --- x-pack/metricbeat/Jenkinsfile.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index e82fd9baef6..77ceb22c157 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -24,6 +24,14 @@ stages: unitTest: mage: "mage build unitTest" stage: mandatory + goIntegTest: + mage: "mage goIntegTest" + withModule: true + stage: mandatory + pythonIntegTest: + mage: "mage pythonIntegTest" + withModule: true + stage: mandatory cloud: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. From cc46a7c3e4ad1e4cfd608b8286c6f47ef5494c37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Mon, 28 Jun 2021 17:50:47 +0200 Subject: [PATCH 08/18] Add changelog entry for #26224 (#26531) --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 48f5b600b76..06c21b8b48f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -844,6 +844,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Journalbeat* +- Suppress too many bad message error logs when reading from corrupted journal for 5 seconds. {pull}26224[26224] *Metricbeat* From a8cb588d1b89b24b48dea974f56803de6ea1db9e Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 28 Jun 2021 13:22:26 -0400 Subject: [PATCH 09/18] [Elastic Agent] Fix issue with FLEET_CA not being used with Fleet Server in container (#26529) * Fix issue with FLEET_CA not being used with fleet server is enabled in container. * Add changelog. --- x-pack/elastic-agent/CHANGELOG.asciidoc | 1 + x-pack/elastic-agent/pkg/agent/cmd/container.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index ba7686a7ccd..0eafcdf2036 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -44,6 +44,7 @@ - Fix windows installer during enroll {pull}24343[24343] - Logging to file disabled on enroll {issue}24173[24173] - Prevent uninstall failures on empty config {pull}24838[24838] +- Fix issue with FLEET_CA not being used with Fleet Server in container {pull}26529[26529] ==== New features diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index 8f7b75578b3..9f69013d41c 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -385,9 +385,9 @@ func buildEnrollArgs(cfg setupConfig, token string, policyID string) ([]string, if cfg.Fleet.Insecure { args = append(args, "--insecure") } - if cfg.Fleet.CA != "" { - args = append(args, "--certificate-authorities", cfg.Fleet.CA) - } + } + if cfg.Fleet.CA != "" { + args = append(args, "--certificate-authorities", cfg.Fleet.CA) } if token != "" { args = append(args, "--enrollment-token", token) From fea3e6c320467ac584bfc17d2c83211d41c7ae65 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Mon, 28 Jun 2021 19:37:19 +0200 Subject: [PATCH 10/18] Skip x-pack metricbeat tests (#26537) --- x-pack/metricbeat/tests/system/test_xpack_base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/metricbeat/tests/system/test_xpack_base.py b/x-pack/metricbeat/tests/system/test_xpack_base.py index ae19590dd24..043540666c2 100644 --- a/x-pack/metricbeat/tests/system/test_xpack_base.py +++ b/x-pack/metricbeat/tests/system/test_xpack_base.py @@ -1,9 +1,11 @@ import os +import unittest import xpack_metricbeat import test_base from beat import common_tests +@unittest.skip("https://github.com/elastic/beats/issues/26536") class Test(xpack_metricbeat.XPackTest, test_base.Test, common_tests.TestExportsMixin): pass From 5667e35a5f367c2efb1bd061971c93f7002b3dac Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 29 Jun 2021 02:01:18 +0800 Subject: [PATCH 11/18] update data.json for gcp billing (#26506) --- .../module/gcp/billing/_meta/data.json | 13 ++++++++----- .../gcp/billing/billing_integration_test.go | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/data.json b/x-pack/metricbeat/module/gcp/billing/_meta/data.json index a9b52f82898..efbb1088bec 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/data.json +++ b/x-pack/metricbeat/module/gcp/billing/_meta/data.json @@ -1,7 +1,8 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "cloud.account.id": "elastic-bi", - "cloud.account.name": "elastic-bi", + "cloud.account.id": "01475F-5B1080-1137E7", + "cloud.project.id": "elastic-bi", + "cloud.project.name": "elastic-containerlib-prod", "cloud.provider": "gcp", "event": { "dataset": "gcp.billing", @@ -10,10 +11,12 @@ }, "gcp": { "billing": { + "billing_account_id": "01475F-5B1080-1137E7", "cost_type": "regular", - "invoice_month": "202010", - "project_id": "elastic-fin-bi", - "total": 77.897328 + "invoice_month": "202106", + "project_id": "containerlib-prod-12763", + "project_name": "elastic-containerlib-prod", + "total": 4717.170681 } }, "metricset": { diff --git a/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go b/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go index 9c006fb30cc..7da63d3dc64 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go +++ b/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go @@ -10,10 +10,27 @@ package billing import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" ) +func TestFetch(t *testing.T) { + config := metrics.GetConfigForTest(t, "billing") + config["period"] = "24h" + config["dataset_id"] = "master_gcp" + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + func TestData(t *testing.T) { config := metrics.GetConfigForTest(t, "billing") config["period"] = "24h" From 0ac76d89619c624775fc09144183e79ee4ddd36f Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Mon, 28 Jun 2021 12:27:48 -0700 Subject: [PATCH 12/18] Remove all docs about Beats central management (#26399) * Remove all docs about Beats central management * Add note about removal of Beats central management --- filebeat/docs/index.asciidoc | 1 - libbeat/docs/security/users.asciidoc | 14 -------------- libbeat/docs/upgrading.asciidoc | 5 +++++ metricbeat/docs/index.asciidoc | 1 - 4 files changed, 5 insertions(+), 16 deletions(-) diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index fc9eaefdc32..0ac2630fb13 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -16,7 +16,6 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :beat_default_index_prefix: {beatname_lc} :beat_kib_app: {kib} Logs :has_ml_jobs: yes -:has_central_config: :has_solutions: :ignores_max_retries: :has_docker_label_ex: diff --git a/libbeat/docs/security/users.asciidoc b/libbeat/docs/security/users.asciidoc index 62d1749eff9..45ccc48710a 100644 --- a/libbeat/docs/security/users.asciidoc +++ b/libbeat/docs/security/users.asciidoc @@ -102,10 +102,6 @@ ifdef::apm-server[] |Set up ingest pipelines endif::apm-server[] -ifdef::has_central_config[] -|`beats_admin` -|Enroll and manage configurations in Beats central management -endif::has_central_config[] |==== + Omit any roles that aren't relevant in your environment. @@ -307,10 +303,6 @@ endif::apm-server[] {kib} users typically need to view dashboards and visualizations that contain {beatname_uc} data. These users might also need to create and edit dashboards and visualizations. -ifdef::has_central_config[] -If you're using Beats central management, some of these users might need to -create and manage configurations. -endif::has_central_config[] To grant users the required privileges: @@ -347,12 +339,6 @@ users who need to read {beatname_uc} data: | `monitoring_user` | Allow users to monitor the health of {beatname_uc} itself. Only assign this role to users who manage {beatname_uc}. -ifdef::has_central_config[] -|`beats_admin` -|Create and manage configurations in Beats central management. Only assign this -role to users who need to use Beats central management. -+ -endif::[] |==== endif::apm-server[] diff --git a/libbeat/docs/upgrading.asciidoc b/libbeat/docs/upgrading.asciidoc index c4557d11071..466d99b9378 100644 --- a/libbeat/docs/upgrading.asciidoc +++ b/libbeat/docs/upgrading.asciidoc @@ -71,6 +71,11 @@ we've provided a migration tool to help you migrate your configurations from version 6.6 to 6.7 or later. For more information, see the https://www.elastic.co/blog/beats-6-7-0-released[Beats 6.7.0 release blog]. +NOTE: {beats} central management has been removed starting in version 7.14.0. +Looking for a replacement? Refer to the +{fleet-guide}/index.html[Fleet User Guide] to learn how to deploy and centrally +manage a single {agent} to monitor and secure each host. + ==== Upgrade {beats} binaries to 7.0 Before upgrading: diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 8fff64854e8..d4801d75c34 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -17,7 +17,6 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :beat_kib_app: {kib} Metrics -:has_central_config: :has_solutions: :has_docker_label_ex: :has_modules_command: From 638b62df72bd8d69cd596a3e55b1f72f63f882ca Mon Sep 17 00:00:00 2001 From: Blake Rouse Date: Mon, 28 Jun 2021 15:57:15 -0400 Subject: [PATCH 13/18] [Elastic Agent] Use http2 to connect to Fleet Server. (#26474) * Use http2 to connect to Fleet Server. * Add changelog. * Fix import formatting. * Fix issue with tls and http2. --- libbeat/common/transport/tls.go | 57 ++++++++++++++++++++ x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + x-pack/elastic-agent/pkg/remote/client.go | 32 ++++++----- 3 files changed, 77 insertions(+), 13 deletions(-) diff --git a/libbeat/common/transport/tls.go b/libbeat/common/transport/tls.go index 10ece84dc47..edef5a6ab9f 100644 --- a/libbeat/common/transport/tls.go +++ b/libbeat/common/transport/tls.go @@ -73,6 +73,63 @@ func TestTLSDialer( }), nil } +type DialerH2 interface { + Dial(network, address string, cfg *tls.Config) (net.Conn, error) +} + +type DialerFuncH2 func(network, address string, cfg *tls.Config) (net.Conn, error) + +func (d DialerFuncH2) Dial(network, address string, cfg *tls.Config) (net.Conn, error) { + return d(network, address, cfg) +} + +func TLSDialerH2(forward Dialer, config *tlscommon.TLSConfig, timeout time.Duration) (DialerH2, error) { + return TestTLSDialerH2(testing.NullDriver, forward, config, timeout) +} + +func TestTLSDialerH2( + d testing.Driver, + forward Dialer, + config *tlscommon.TLSConfig, + timeout time.Duration, +) (DialerH2, error) { + var lastTLSConfig *tls.Config + var lastNetwork string + var lastAddress string + var m sync.Mutex + + return DialerFuncH2(func(network, address string, cfg *tls.Config) (net.Conn, error) { + switch network { + case "tcp", "tcp4", "tcp6": + default: + return nil, fmt.Errorf("unsupported network type %v", network) + } + + host, _, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + + var tlsConfig *tls.Config + m.Lock() + if network == lastNetwork && address == lastAddress { + tlsConfig = lastTLSConfig + } + if tlsConfig == nil { + tlsConfig = config.BuildModuleClientConfig(host) + lastNetwork = network + lastAddress = address + lastTLSConfig = tlsConfig + } + m.Unlock() + + // NextProtos must be set from the passed h2 connection or it will fail + tlsConfig.NextProtos = cfg.NextProtos + + return tlsDialWith(d, forward, network, address, timeout, tlsConfig, config) + }), nil +} + func tlsDialWith( d testing.Driver, dialer Dialer, diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index d54b3e273a9..d5eed75dc0f 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -115,3 +115,4 @@ - Use `filestream` input for internal log collection. {pull}25660[25660] - Enable agent to send custom headers to kibana/ES {pull}26275[26275] - Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394] +- Communicate with Fleet Server over HTTP2. {pull}26474[26474] diff --git a/x-pack/elastic-agent/pkg/remote/client.go b/x-pack/elastic-agent/pkg/remote/client.go index 281b3798944..ad5f136f7e0 100644 --- a/x-pack/elastic-agent/pkg/remote/client.go +++ b/x-pack/elastic-agent/pkg/remote/client.go @@ -15,6 +15,7 @@ import ( "time" "github.com/pkg/errors" + "golang.org/x/net/http2" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/transport" @@ -113,8 +114,16 @@ func NewWithConfig(log *logger.Logger, cfg Config, wrapper wrapperFunc) (*Client hosts := cfg.GetHosts() clients := make([]*requestClient, len(hosts)) for i, host := range cfg.GetHosts() { - var transport http.RoundTripper - transport, err := makeTransport(cfg.Timeout, cfg.TLS) + connStr, err := common.MakeURL(string(cfg.Protocol), p, host, 0) + if err != nil { + return nil, errors.Wrap(err, "invalid fleet-server endpoint") + } + addr, err := url.Parse(connStr) + if err != nil { + return nil, errors.Wrap(err, "invalid fleet-server endpoint") + } + + transport, err := makeTransport(addr.Scheme, cfg.Timeout, cfg.TLS) if err != nil { return nil, err } @@ -136,12 +145,8 @@ func NewWithConfig(log *logger.Logger, cfg Config, wrapper wrapperFunc) (*Client Timeout: cfg.Timeout, } - url, err := common.MakeURL(string(cfg.Protocol), p, host, 0) - if err != nil { - return nil, errors.Wrap(err, "invalid fleet-server endpoint") - } clients[i] = &requestClient{ - request: prefixRequestFactory(url), + request: prefixRequestFactory(connStr), client: httpClient, } } @@ -272,17 +277,18 @@ func prefixRequestFactory(URL string) requestFunc { } // makeTransport create a transport object based on the TLS configuration. -func makeTransport(timeout time.Duration, tls *tlscommon.Config) (*http.Transport, error) { +func makeTransport(scheme string, timeout time.Duration, tls *tlscommon.Config) (http.RoundTripper, error) { + dialer := transport.NetDialer(timeout) + if scheme == "http" { + return &http.Transport{Dial: dialer.Dial}, nil + } tlsConfig, err := tlscommon.LoadTLSConfig(tls) if err != nil { return nil, errors.Wrap(err, "invalid TLS configuration") } - dialer := transport.NetDialer(timeout) - tlsDialer, err := transport.TLSDialer(dialer, tlsConfig, timeout) + tlsDialer, err := transport.TLSDialerH2(dialer, tlsConfig, timeout) if err != nil { return nil, errors.Wrap(err, "fail to create TLS dialer") } - - // TODO: Dial is deprecated we need to move to DialContext. - return &http.Transport{Dial: dialer.Dial, DialTLS: tlsDialer.Dial}, nil + return &http2.Transport{DialTLS: tlsDialer.Dial}, nil } From 7ca5909d70808a8d007672468c31ee8c54d58427 Mon Sep 17 00:00:00 2001 From: Vignesh Shanmugam Date: Mon, 28 Jun 2021 13:30:57 -0700 Subject: [PATCH 14/18] [Heartbeat] add screenshots config to synthetics (#26455) * [Heartbeat] add screenshots config to synthetics * docs: update screenshot docs --- .../docs/monitors/monitor-browser.asciidoc | 31 ++++++++++++++++++- x-pack/heartbeat/monitors/browser/config.go | 4 ++- x-pack/heartbeat/monitors/browser/suite.go | 3 ++ .../heartbeat/monitors/browser/suite_test.go | 10 ++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/heartbeat/docs/monitors/monitor-browser.asciidoc b/heartbeat/docs/monitors/monitor-browser.asciidoc index 69ce06f88ca..ab2e0cb85ff 100644 --- a/heartbeat/docs/monitors/monitor-browser.asciidoc +++ b/heartbeat/docs/monitors/monitor-browser.asciidoc @@ -115,4 +115,33 @@ Set this option to `true` to enable the normally disabled chromium sandbox. Defa [[monitor-browser-synthetics-args]] ==== `synthetics_args` -Extra arguments to pass to the synthetics agent package. Takes a list of strings. \ No newline at end of file +Extra arguments to pass to the synthetics agent package. Takes a list of +strings. + +[float] +[[monitor-browser-screenshots]] +==== `screenshots` + +Set this option to manage the screenshots captured by the synthetics agent. + +Under `screenshots`, specify one of these options: + +*`on`*:: capture screenshots for all steps in a journey (default) +*`off`*:: do not capture any screenshots +*`only-on-failure`*:: capture screenshots for all steps when a journey fails +(any failing step marks the whole journey as failed) + +Example configuration: + +[source,yaml] +------------------------------------------------------------------------------- +- type: browser + id: local-journeys + name: Local journeys + schedule: '@every 1m' + screenshots: "on" + source: + local: + path: "/path/to/synthetics/journeys" +------------------------------------------------------------------------------- + diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go index 0da4eb0a860..3be6af640ac 100644 --- a/x-pack/heartbeat/monitors/browser/config.go +++ b/x-pack/heartbeat/monitors/browser/config.go @@ -13,7 +13,8 @@ import ( func DefaultConfig() *Config { return &Config{ - Sandbox: false, + Sandbox: false, + Screenshots: "on", } } @@ -27,6 +28,7 @@ type Config struct { // Id is optional for lightweight checks but required for browsers Id string `config:"id"` Sandbox bool `config:"sandbox"` + Screenshots string `config:"screenshots"` SyntheticsArgs []string `config:"synthetics_args"` } diff --git a/x-pack/heartbeat/monitors/browser/suite.go b/x-pack/heartbeat/monitors/browser/suite.go index d265c0a4b3f..5e3d286d460 100644 --- a/x-pack/heartbeat/monitors/browser/suite.go +++ b/x-pack/heartbeat/monitors/browser/suite.go @@ -75,6 +75,9 @@ func (s *Suite) extraArgs() []string { if s.suiteCfg.Sandbox { extraArgs = append(extraArgs, "--sandbox") } + if s.suiteCfg.Screenshots != "" { + extraArgs = append(extraArgs, "--screenshots", s.suiteCfg.Screenshots) + } return extraArgs } diff --git a/x-pack/heartbeat/monitors/browser/suite_test.go b/x-pack/heartbeat/monitors/browser/suite_test.go index c583492d39d..54e708c3e07 100644 --- a/x-pack/heartbeat/monitors/browser/suite_test.go +++ b/x-pack/heartbeat/monitors/browser/suite_test.go @@ -127,11 +127,21 @@ func TestExtraArgs(t *testing.T) { &Config{}, nil, }, + { + "default", + DefaultConfig(), + []string{"--screenshots", "on"}, + }, { "sandbox", &Config{Sandbox: true}, []string{"--sandbox"}, }, + { + "screenshots", + &Config{Screenshots: "off"}, + []string{"--screenshots", "off"}, + }, { "capabilities", &Config{SyntheticsArgs: []string{"--capability", "trace", "ssblocks"}}, From 67cf2c6c341944fd127ec533324979560223cbaa Mon Sep 17 00:00:00 2001 From: Aleksandr Maus Date: Mon, 28 Jun 2021 13:59:29 -0700 Subject: [PATCH 15/18] Osquerybeat: set the raw index name to supress the timestamp suffix (#26545) --- x-pack/osquerybeat/beater/osquerybeat.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/osquerybeat/beater/osquerybeat.go b/x-pack/osquerybeat/beater/osquerybeat.go index fef59f975ec..24a24623c08 100644 --- a/x-pack/osquerybeat/beater/osquerybeat.go +++ b/x-pack/osquerybeat/beater/osquerybeat.go @@ -19,6 +19,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" @@ -430,7 +431,7 @@ func (bt *osquerybeat) publishEvents(index, actionID, responseID string, hits [] event.Fields["response_id"] = responseID } if index != "" { - event.Meta = common.MapStr{"index": index} + event.Meta = common.MapStr{events.FieldMetaRawIndex: index} } bt.client.Publish(event) From 591fc3a7c9485b94a0631afe4ca6afaedc89f2f9 Mon Sep 17 00:00:00 2001 From: Vignesh Shanmugam Date: Tue, 29 Jun 2021 00:55:17 -0700 Subject: [PATCH 16/18] [Heartbeat] configure permissions for synthetics config (#26393) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- dev-tools/packaging/templates/docker/Dockerfile.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 880aac57ccd..dd8bd021a8c 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -47,7 +47,7 @@ ENV NODE_PATH={{ $beatHome }}/.node RUN echo \ $NODE_PATH \ {{ $beatHome }}/.config \ - {{ $beatHome }}/suites \ + {{ $beatHome }}/.synthetics \ {{ $beatHome }}/.npm \ {{ $beatHome }}/.cache \ | xargs -IDIR sh -c 'mkdir -p DIR && chmod 0770 DIR' From bba9cfd70e6c1c9ef6ab7ca215d01567f731fed2 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 29 Jun 2021 10:12:20 +0200 Subject: [PATCH 17/18] [Elastic Agent] Enable configuring monitoring namespace (#26439) [Elastic Agent] Enable configuring monitoring namespace (#26439) --- x-pack/elastic-agent/CHANGELOG.next.asciidoc | 1 + .../emitter/modifiers/monitoring_decorator.go | 1 + .../pkg/agent/operation/monitoring.go | 33 +-- .../pkg/agent/operation/monitoring_test.go | 3 + .../pkg/agent/program/program_test.go | 4 + .../testdata/namespace-endpoint-security.yml | 114 ++++++++++ .../program/testdata/namespace-filebeat.yml | 68 ++++++ .../testdata/namespace-fleet-server.yml | 16 ++ .../program/testdata/namespace-heartbeat.yml | 30 +++ .../program/testdata/namespace-metricbeat.yml | 88 ++++++++ .../program/testdata/namespace-packetbeat.yml | 35 +++ .../pkg/agent/program/testdata/namespace.yml | 201 ++++++++++++++++++ .../core/monitoring/beats/beats_monitor.go | 9 + .../pkg/core/monitoring/config/config.go | 3 + .../pkg/core/monitoring/monitor.go | 1 + .../pkg/core/monitoring/noop/noop_monitor.go | 3 + 16 files changed, 594 insertions(+), 16 deletions(-) create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml create mode 100644 x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index d5eed75dc0f..7fe323a0b64 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -115,4 +115,5 @@ - Use `filestream` input for internal log collection. {pull}25660[25660] - Enable agent to send custom headers to kibana/ES {pull}26275[26275] - Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394] +- Enable configuring monitoring namespace {issue}26439[26439] - Communicate with Fleet Server over HTTP2. {pull}26474[26474] diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go index 8c3eb1c7d43..b31220f93de 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go @@ -49,6 +49,7 @@ func InjectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr transpiler.NewKey("logs", transpiler.NewBoolVal(true)), transpiler.NewKey("metrics", transpiler.NewBoolVal(true)), transpiler.NewKey("use_output", transpiler.NewStrVal("default")), + transpiler.NewKey("namespace", transpiler.NewStrVal("default")), }) transpiler.Insert(rootAst, transpiler.NewKey("monitoring", monitoringNode), "settings") diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 45b7263cf73..17188321b56 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -161,11 +161,12 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in var steps []configrequest.Step watchLogs := o.monitor.WatchLogs() watchMetrics := o.monitor.WatchMetrics() + monitoringNamespace := o.monitor.MonitoringNamespace() // generate only when monitoring is running (for config refresh) or // state changes (turning on/off) if watchLogs != o.isMonitoringLogs() || watchLogs { - fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output) + fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output, monitoringNamespace) stepID := configrequest.StepRun if !watchLogs || !any { stepID = configrequest.StepRemove @@ -182,7 +183,7 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in steps = append(steps, filebeatStep) } if watchMetrics != o.isMonitoringMetrics() || watchMetrics { - mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output) + mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output, monitoringNamespace) stepID := configrequest.StepRun if !watchMetrics || !any { stepID = configrequest.StepRemove @@ -215,12 +216,12 @@ func loadSpecFromSupported(processName string) program.Spec { } } -func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { inputs := []interface{}{ map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -233,7 +234,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"), filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"), }, - "index": "logs-elastic_agent-default", + "index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -241,7 +242,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa "fields": map[string]interface{}{ "type": "logs", "dataset": "elastic_agent", - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -280,7 +281,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa inputs = append(inputs, map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -288,7 +289,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa }, }, "paths": paths, - "index": fmt.Sprintf("logs-elastic_agent.%s-default", name), + "index": fmt.Sprintf("logs-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -296,7 +297,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa "fields": map[string]interface{}{ "type": "logs", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -345,7 +346,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa return result, true } -func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { hosts := o.getMetricbeatEndpoints() if len(hosts) == 0 { return nil, false @@ -359,7 +360,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "metricsets": []string{"stats", "state"}, "period": "10s", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", name), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -367,7 +368,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -397,7 +398,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "period": "10s", "path": "/stats", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -405,7 +406,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -480,7 +481,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "period": "10s", "path": "/stats", "hosts": []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS(), o.config.MonitoringConfig.HTTP)}, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -488,7 +489,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go index 136c9e485b1..c23248ff2d9 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go @@ -215,6 +215,9 @@ func (b *testMonitor) Reload(cfg *config.Config) error { return nil } // IsMonitoringEnabled returns true if monitoring is configured. func (b *testMonitor) IsMonitoringEnabled() bool { return b.monitorLogs || b.monitorMetrics } +// MonitoringNamespace returns monitoring namespace configured. +func (b *testMonitor) MonitoringNamespace() string { return "default" } + // WatchLogs return true if monitoring is configured and monitoring logs is enabled. func (b *testMonitor) WatchLogs() bool { return b.monitorLogs } diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index 4498f7e5236..5ca35de0136 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -383,6 +383,10 @@ func TestConfiguration(t *testing.T) { empty bool err bool }{ + "namespace": { + programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, + expected: 6, + }, "single_config": { programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, expected: 6, diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml new file mode 100644 index 00000000000..7e9f04dc411 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml @@ -0,0 +1,114 @@ +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: + - "127.0.0.1:9200" + - "127.0.0.1:9300" + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + +inputs: +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml new file mode 100644 index 00000000000..83df83e56e0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml @@ -0,0 +1,68 @@ +filebeat: + inputs: + - type: log + paths: + - /var/log/hello1.log + - /var/log/hello2.log + index: logs-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: logs + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - type: log + paths: + - /var/log/hello3.log + - /var/log/hello4.log + index: testtype-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: testtype + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml new file mode 100644 index 00000000000..c03696aff1f --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml @@ -0,0 +1,16 @@ +fleet: + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +inputs: + - id: fleet-server-id + type: fleet-server diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml new file mode 100644 index 00000000000..f34b204f5fa --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml @@ -0,0 +1,30 @@ +inputs: +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s + data_stream.namespace: default + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml new file mode 100644 index 00000000000..3f16a9d9e21 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml @@ -0,0 +1,88 @@ +metricbeat: + modules: + - module: docker + metricsets: [status] + index: metrics-docker.status-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: docker.status + namespace: default + - add_fields: + target: "event" + fields: + dataset: docker.status + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: docker + metricsets: [info] + index: metrics-generic-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: apache + metricsets: [info] + index: metrics-generic-testing + hosts: ["http://apache.remote"] + processors: + - add_fields: + fields: + should_be: first + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: testing + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml new file mode 100644 index 00000000000..d71499bdef4 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml @@ -0,0 +1,35 @@ +inputs: +- type: packet + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml new file mode 100644 index 00000000000..c2f83a9abf0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml @@ -0,0 +1,201 @@ +name: Production Website DB Servers +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + server: + output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +outputs: + default: + type: elasticsearch + namespace: test_namespace + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + + monitoring: + type: elasticsearch + api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + hosts: ["monitoring:9200"] + ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M=" + +inputs: +- id: fleet-server-id + type: fleet-server + use_output: default + data_stream: + type: default +- type: docker/metrics + use_output: default + streams: + - metricset: status + processors: + - null + data_stream: + dataset: docker.status + - metricset: info + data_stream: + dataset: "" + hosts: ["http://127.0.0.1:8080"] +- type: logfile + use_output: default + streams: + - paths: + - /var/log/hello1.log + - /var/log/hello2.log + vars: + var: value +- type: logfile + data_stream: + type: testtype + use_output: default + streams: + - paths: + - /var/log/hello3.log + - /var/log/hello4.log + vars: + var: value +- id: apache-metrics-id + type: apache/metrics + data_stream: + namespace: testing + use_output: default + processors: + - add_fields: + fields: + should_be: first + streams: + - enabled: true + metricset: info + hosts: ["http://apache.remote"] + hosts: ["http://apache.local"] +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s +- type: packet + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + +agent.monitoring: + use_output: monitoring + +agent: + reload: 123 diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index 743d44118d6..1c0c4ba61ad 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -21,6 +21,7 @@ import ( ) const httpPlusPrefix = "http+" +const defaultMonitoringNamespace = "default" // Monitor is a monitoring interface providing information about the way // how beat is monitored @@ -69,6 +70,14 @@ func (b *Monitor) Close() { // IsMonitoringEnabled returns true if monitoring is enabled. func (b *Monitor) IsMonitoringEnabled() bool { return b.config.Enabled } +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { + if b.config.Namespace == "" { + return defaultMonitoringNamespace + } + return b.config.Namespace +} + // WatchLogs returns true if monitoring is enabled and monitor should watch logs. func (b *Monitor) WatchLogs() bool { return b.config.Enabled && b.config.MonitorLogs } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go index 2ce067d4e19..fe18b0fb73e 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go @@ -5,6 +5,7 @@ package config const defaultPort = 6791 +const defaultNamespace = "default" // MonitoringConfig describes a configuration of a monitoring type MonitoringConfig struct { @@ -12,6 +13,7 @@ type MonitoringConfig struct { MonitorLogs bool `yaml:"logs" config:"logs"` MonitorMetrics bool `yaml:"metrics" config:"metrics"` HTTP *MonitoringHTTPConfig `yaml:"http" config:"http"` + Namespace string `yaml:"namespace" config:"namespace"` } // MonitoringHTTPConfig is a config defining HTTP endpoint published by agent @@ -33,5 +35,6 @@ func DefaultConfig() *MonitoringConfig { Enabled: false, Port: defaultPort, }, + Namespace: defaultNamespace, } } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go index 00c7a50003a..6c71f4f65fc 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go @@ -23,6 +23,7 @@ type Monitor interface { Cleanup(spec program.Spec, pipelineID string) error Reload(cfg *config.Config) error IsMonitoringEnabled() bool + MonitoringNamespace() string WatchLogs() bool WatchMetrics() bool Close() diff --git a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go index 9ea8f08a788..d98deb90888 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go @@ -66,3 +66,6 @@ func (b *Monitor) WatchLogs() bool { return false } // WatchMetrics return true if monitoring is configured and monitoring metrics is enabled. func (b *Monitor) WatchMetrics() bool { return false } + +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { return "default" } From b17daafded2fd0dbc832855732c29c7133d563cd Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Tue, 29 Jun 2021 10:12:54 +0200 Subject: [PATCH 18/18] [Elastic Agent] Improper casting of int64 (#26520) --- x-pack/elastic-agent/pkg/agent/transpiler/ast.go | 4 +++- x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go index 100d8a462a0..31bb2faaa7c 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go @@ -716,8 +716,10 @@ func load(val reflect.Value) (Node, error) { return loadSliceOrArray(val) case reflect.String: return &StrVal{value: val.Interface().(string)}, nil - case reflect.Int, reflect.Int64: + case reflect.Int: return &IntVal{value: val.Interface().(int)}, nil + case reflect.Int64: + return &IntVal{value: int(val.Interface().(int64))}, nil case reflect.Uint: return &UIntVal{value: uint64(val.Interface().(uint))}, nil case reflect.Uint64: diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go index e1b22c390ed..1e52d6c6f47 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go @@ -105,6 +105,7 @@ func TestAST(t *testing.T) { "support integers": { hashmap: map[string]interface{}{ "timeout": 12, + "zero": int64(0), "range": []int{20, 30, 40}, }, ast: &AST{ @@ -121,6 +122,7 @@ func TestAST(t *testing.T) { ), }, &Key{name: "timeout", value: &IntVal{value: 12}}, + &Key{name: "zero", value: &IntVal{value: 0}}, }, }, },