diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 8eb9f5bb168..5e4f3074799 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -236,6 +236,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143] - Omit full index template from errors that occur while loading the template. {pull}25743[25743] - In the script processor, the `decode_xml` and `decode_xml_wineventlog` processors are now available as `DecodeXML` and `DecodeXMLWineventlog` respectively. +- Fix encoding errors when using the disk queue on nested data with multi-byte characters {pull}26484[26484] *Auditbeat* @@ -819,6 +820,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686] - Support MongoDB 4.4 in filebeat's MongoDB module. {issue}20501[20501] {pull}24774[24774] - Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368] +- Add log_group_name_prefix config into aws-cloudwatch input. {pull}26187[26187] - Move Filebeat azure module to GA. {pull}26114[26114] {pull}26168[26168] - http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764] - Make `filestream` input GA. {pull}26127[26127] @@ -835,6 +837,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350] - Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457] - Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441] +- Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481] *Heartbeat* @@ -843,6 +846,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d *Journalbeat* +- Suppress too many bad message error logs when reading from corrupted journal for 5 seconds. {pull}26224[26224] *Metricbeat* diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 880aac57ccd..dd8bd021a8c 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -47,7 +47,7 @@ ENV NODE_PATH={{ $beatHome }}/.node RUN echo \ $NODE_PATH \ {{ $beatHome }}/.config \ - {{ $beatHome }}/suites \ + {{ $beatHome }}/.synthetics \ {{ $beatHome }}/.npm \ {{ $beatHome }}/.cache \ | xargs -IDIR sh -c 'mkdir -p DIR && chmod 0770 DIR' diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index c983f91aca6..099296a099e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -152208,6 +152208,17 @@ example: Montreal -- +*`threatintel.indicator.geo.continent_name`*:: ++ +-- +Name of the continent. + +type: keyword + +example: North America + +-- + *`threatintel.indicator.geo.country_iso_code`*:: + -- @@ -153643,6 +153654,166 @@ type: keyword -- +[float] +=== recordedfuture + +Fields for Recorded Future Threat Intel + + + +[float] +=== entity + +Entity that represents a threat. + + + +*`threatintel.recordedfuture.entity.id`*:: ++ +-- +Entity ID. + + +type: keyword + +example: ip:192.0.2.13 + +-- + +*`threatintel.recordedfuture.entity.name`*:: ++ +-- +Entity name. Value for the entity. + + +type: keyword + +example: 192.0.2.13 + +-- + +*`threatintel.recordedfuture.entity.type`*:: ++ +-- +Entity type. + + +type: keyword + +example: IpAddress + +-- + +*`threatintel.recordedfuture.intelCard`*:: ++ +-- +Link to the Recorded Future Intelligence Card for to this indicator. + + +type: keyword + +-- + +*`threatintel.recordedfuture.ip_range`*:: ++ +-- +Range of IPs for this indicator. + + +type: ip_range + +example: 192.0.2.0/16 + +-- + +[float] +=== risk + +Risk fields. + + + +*`threatintel.recordedfuture.risk.criticality`*:: ++ +-- +Risk criticality (0-4). + + +type: byte + +-- + +*`threatintel.recordedfuture.risk.criticalityLabel`*:: ++ +-- +Risk criticality label. One of None, Unusual, Suspicious, Malicious, Very Malicious. + + +type: keyword + +-- + +*`threatintel.recordedfuture.risk.evidenceDetails`*:: ++ +-- +Risk's evidence details. + + +type: flattened + +-- + +*`threatintel.recordedfuture.risk.score`*:: ++ +-- +Risk score (0-99). + + +type: short + +-- + +*`threatintel.recordedfuture.risk.riskString`*:: ++ +-- +Number of Risk Rules observed as a factor of total number of rules. + + +type: keyword + +example: 1/54 + +-- + +*`threatintel.recordedfuture.risk.riskSummary`*:: ++ +-- +Risk summary. + + +type: keyword + +example: 1 of 54 Risk Rules currently observed. + +-- + +*`threatintel.recordedfuture.risk.riskSummary.text`*:: ++ +-- +type: text + +-- + +*`threatintel.recordedfuture.risk.rules`*:: ++ +-- +Number of rules observed. + + +type: long + +-- + [[exported-fields-tomcat]] == Apache Tomcat fields diff --git a/filebeat/docs/index.asciidoc b/filebeat/docs/index.asciidoc index fc9eaefdc32..0ac2630fb13 100644 --- a/filebeat/docs/index.asciidoc +++ b/filebeat/docs/index.asciidoc @@ -16,7 +16,6 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :beat_default_index_prefix: {beatname_lc} :beat_kib_app: {kib} Logs :has_ml_jobs: yes -:has_central_config: :has_solutions: :ignores_max_retries: :has_docker_label_ex: diff --git a/filebeat/docs/modules/threatintel.asciidoc b/filebeat/docs/modules/threatintel.asciidoc index f39a9377fb1..fd3be0d93f3 100644 --- a/filebeat/docs/modules/threatintel.asciidoc +++ b/filebeat/docs/modules/threatintel.asciidoc @@ -29,6 +29,7 @@ The available filesets are: * <>: Supports gathering threat intel attributes from AlientVault OTX. * <>: Supports gathering threat intel attributes from Anomali Limo. * <>: Supports gathering threat intel attributes from Anomali ThreatStream. +* <>: Supports gathering threat intel attributes from Recorded Future. include::../include/gs-link.asciidoc[] @@ -223,7 +224,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -297,7 +298,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time the {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -409,7 +410,7 @@ Anomali Threat Intel is mapped to the following ECS fields. To configure the ThreatStream integration you first need to define an output in the Anomali ThreatStream Integrator using the Elastic SDK provided by Anomali. -It will deliver indicators via HTTP or HTTPS to a Filebeat instance running as +It will deliver indicators via HTTP or HTTPS to a {beatname_uc} instance running as a server. Configure an Integrator output with the following settings: @@ -419,12 +420,12 @@ Configure an Integrator output with the following settings: Adjust the paths to the python executable and the directory where the Elastic SDK has been unpacked. * Metadata in JSON Format: `{"url": "https://filebeat:8080/", "server_certificate": "/path/to/cert.pem", "secret": "my secret"}`. - - `url`: Use the host and port where Filebeat will be running, and `http` or `https` accordingly. + - `url`: Use the host and port where {beatname_uc} will be running, and `http` or `https` accordingly. - `server_certificate`: If using HTTPS, absolute path to the server certificate. Otherwise don't set this field. - - `secret`: A shared secret string to authenticate messages between the SDK and Filebeat. + - `secret`: A shared secret string to authenticate messages between the SDK and {beatname_uc}. -Then configure the `anomalithreatstream` fileset in Filebeat accordingly: +Then configure the `anomalithreatstream` fileset in {beatname_uc} accordingly: [source,yaml] ---- - module: threatintel @@ -449,11 +450,11 @@ Port number to use for the HTTP server. *`var.secret`*:: -Shared secret between the SDK and Filebeat, used to authenticate messages. +Shared secret between the SDK and {beatname_uc}, used to authenticate messages. *`var.ssl_certificate`*:: -Path to the public SSL certificate for the HTTPS server. If unset, Filebeat +Path to the public SSL certificate for the HTTPS server. If unset, {beatname_uc} will use unsecure HTTP connections. *`var.ssl_key`*:: @@ -488,6 +489,94 @@ Anomali ThreatStream fields are mapped to the following ECS fields: [[a]] [small]#[1]: Field is used to derive a value for the ECS field but its original value is kept under `threatintel.anomalithreatstream`.# +[[recordedfuture]] +[float] +==== `recordedfuture` fileset settings + +The `recordedfuture` fileset fetches intelligence from the Recorded Future Connect API. +It supports `domain`, `hash`, `ip` and `url` data types. + +To enable it you need to define the URL to fetch data from. You can construct this URL +using the https://api.recordedfuture.com/index.html[Recorded Future API Explorer.] The URL +must point to the `/search` endpoint and contain a suitable `limit` +(how many records to return from a single request) and `fields` parameters. +The `entity` and `timestamps` fields are required. + +Sample configuration: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +To fetch threat intelligence from multiple data types, you must define more than +one instance of the module: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 1m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +*`var.url`*:: + +The URL of the API endpoint to connect with. + +*`var.api_token`*:: + +The API token used to access Recorded Future API. + +*`var.interval`*:: + +How often the API is polled for updated information. + +*`var.first_interval`*:: + +How far back to search when retrieving events the first time {beatname_uc} starts up. +After the first interval has passed the module itself will use the timestamp +from the last response as the filter when retrieving new events. + +*`var.proxy_url`*:: + +Optional URL to use as HTTP proxy. + + +Recorded Future fields are mapped to the following ECS fields: + +[options="header"] +|============================================================= +| Recorded Future fields | ECS Fields +| entity.name | threatintel.indicator.{url,ip,domain,file.hash} +| entity.type | threatintel.indicator.type +| fileHashes | threatintel.indicator.file.hash +| intelCard | event.reference +| location.asn | threatintel.indicator.as.number +| location.location | threatintel.indicator.geo +| location.organization | threatintel.indicator.as.organization.name +| risk.score | event.risk_score +| timestamps.firstSeen | threatintel.indicator.first_seen +| timestamps.lastSeen | threatintel.indicator.last_seen +|============================================================= + :has-dashboards!: [float] diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py index fa8507a5952..13f9ac899fc 100644 --- a/filebeat/tests/system/test_modules.py +++ b/filebeat/tests/system/test_modules.py @@ -279,6 +279,7 @@ def clean_keys(obj): "threatintel.anomali", "threatintel.anomalithreatstream", "threatintel.malwarebazaar", + "threatintel.recordedfuture", "snyk.vulnerabilities", "snyk.audit", "awsfargate.log", diff --git a/heartbeat/docs/monitors/monitor-browser.asciidoc b/heartbeat/docs/monitors/monitor-browser.asciidoc index 69ce06f88ca..ab2e0cb85ff 100644 --- a/heartbeat/docs/monitors/monitor-browser.asciidoc +++ b/heartbeat/docs/monitors/monitor-browser.asciidoc @@ -115,4 +115,33 @@ Set this option to `true` to enable the normally disabled chromium sandbox. Defa [[monitor-browser-synthetics-args]] ==== `synthetics_args` -Extra arguments to pass to the synthetics agent package. Takes a list of strings. \ No newline at end of file +Extra arguments to pass to the synthetics agent package. Takes a list of +strings. + +[float] +[[monitor-browser-screenshots]] +==== `screenshots` + +Set this option to manage the screenshots captured by the synthetics agent. + +Under `screenshots`, specify one of these options: + +*`on`*:: capture screenshots for all steps in a journey (default) +*`off`*:: do not capture any screenshots +*`only-on-failure`*:: capture screenshots for all steps when a journey fails +(any failing step marks the whole journey as failed) + +Example configuration: + +[source,yaml] +------------------------------------------------------------------------------- +- type: browser + id: local-journeys + name: Local journeys + schedule: '@every 1m' + screenshots: "on" + source: + local: + path: "/path/to/synthetics/journeys" +------------------------------------------------------------------------------- + diff --git a/journalbeat/input/input.go b/journalbeat/input/input.go index b45b99d1816..f1238d61c81 100644 --- a/journalbeat/input/input.go +++ b/journalbeat/input/input.go @@ -18,12 +18,18 @@ package input import ( + "context" "fmt" + "strings" "sync" + "syscall" + "time" "github.com/elastic/beats/v7/libbeat/processors/add_formatted_index" + "github.com/elastic/go-concert/timed" "github.com/elastic/beats/v7/libbeat/common/acker" + "github.com/elastic/beats/v7/libbeat/common/atomic" "github.com/elastic/beats/v7/libbeat/common/fmtstr" "github.com/elastic/beats/v7/journalbeat/checkpoint" @@ -158,6 +164,10 @@ func (i *Input) publishAll() { go func() { defer wg.Done() + suppressed := atomic.NewBool(false) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for { select { case <-i.done: @@ -168,6 +178,10 @@ func (i *Input) publishAll() { event, err := r.Next() if event == nil { if err != nil { + if i.isErrSuppressed(ctx, err, suppressed) { + i.logger.Debugf("Error message suppressed: EBADMSG") + continue + } i.logger.Errorf("Error while reading event: %v", err) } continue @@ -191,6 +205,26 @@ func (i *Input) publishAll() { } } +// isErrSuppressed checks if the error is due to a corrupt journal. If yes, only the first error message +// is displayed and then it is suppressed for 5 seconds. +func (i *Input) isErrSuppressed(ctx context.Context, err error, suppressed *atomic.Bool) bool { + if strings.Contains(err.Error(), syscall.EBADMSG.Error()) { + if suppressed.Load() { + return true + } + + suppressed.Store(true) + go func(ctx context.Context, suppressed *atomic.Bool) { + if err := timed.Wait(ctx, 5*time.Second); err == nil { + suppressed.Store(false) + } + + }(ctx, suppressed) + } + + return false +} + // Stop stops all readers of the input. func (i *Input) Stop() { for _, r := range i.readers { diff --git a/libbeat/common/transport/tls.go b/libbeat/common/transport/tls.go index a083fec2da2..5f8ade67012 100644 --- a/libbeat/common/transport/tls.go +++ b/libbeat/common/transport/tls.go @@ -73,6 +73,63 @@ func TestTLSDialer( }) } +type DialerH2 interface { + Dial(network, address string, cfg *tls.Config) (net.Conn, error) +} + +type DialerFuncH2 func(network, address string, cfg *tls.Config) (net.Conn, error) + +func (d DialerFuncH2) Dial(network, address string, cfg *tls.Config) (net.Conn, error) { + return d(network, address, cfg) +} + +func TLSDialerH2(forward Dialer, config *tlscommon.TLSConfig, timeout time.Duration) (DialerH2, error) { + return TestTLSDialerH2(testing.NullDriver, forward, config, timeout) +} + +func TestTLSDialerH2( + d testing.Driver, + forward Dialer, + config *tlscommon.TLSConfig, + timeout time.Duration, +) (DialerH2, error) { + var lastTLSConfig *tls.Config + var lastNetwork string + var lastAddress string + var m sync.Mutex + + return DialerFuncH2(func(network, address string, cfg *tls.Config) (net.Conn, error) { + switch network { + case "tcp", "tcp4", "tcp6": + default: + return nil, fmt.Errorf("unsupported network type %v", network) + } + + host, _, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + + var tlsConfig *tls.Config + m.Lock() + if network == lastNetwork && address == lastAddress { + tlsConfig = lastTLSConfig + } + if tlsConfig == nil { + tlsConfig = config.BuildModuleClientConfig(host) + lastNetwork = network + lastAddress = address + lastTLSConfig = tlsConfig + } + m.Unlock() + + // NextProtos must be set from the passed h2 connection or it will fail + tlsConfig.NextProtos = cfg.NextProtos + + return tlsDialWith(d, forward, network, address, timeout, tlsConfig, config) + }), nil +} + func tlsDialWith( d testing.Driver, dialer Dialer, diff --git a/libbeat/docs/security/users.asciidoc b/libbeat/docs/security/users.asciidoc index 62d1749eff9..45ccc48710a 100644 --- a/libbeat/docs/security/users.asciidoc +++ b/libbeat/docs/security/users.asciidoc @@ -102,10 +102,6 @@ ifdef::apm-server[] |Set up ingest pipelines endif::apm-server[] -ifdef::has_central_config[] -|`beats_admin` -|Enroll and manage configurations in Beats central management -endif::has_central_config[] |==== + Omit any roles that aren't relevant in your environment. @@ -307,10 +303,6 @@ endif::apm-server[] {kib} users typically need to view dashboards and visualizations that contain {beatname_uc} data. These users might also need to create and edit dashboards and visualizations. -ifdef::has_central_config[] -If you're using Beats central management, some of these users might need to -create and manage configurations. -endif::has_central_config[] To grant users the required privileges: @@ -347,12 +339,6 @@ users who need to read {beatname_uc} data: | `monitoring_user` | Allow users to monitor the health of {beatname_uc} itself. Only assign this role to users who manage {beatname_uc}. -ifdef::has_central_config[] -|`beats_admin` -|Create and manage configurations in Beats central management. Only assign this -role to users who need to use Beats central management. -+ -endif::[] |==== endif::apm-server[] diff --git a/libbeat/docs/shared-deduplication.asciidoc b/libbeat/docs/shared-deduplication.asciidoc index eec45caaf97..e0d1f06e1aa 100644 --- a/libbeat/docs/shared-deduplication.asciidoc +++ b/libbeat/docs/shared-deduplication.asciidoc @@ -114,7 +114,7 @@ input { beats { port => 5044 } -}} +} output { if [@metadata][_id] { diff --git a/libbeat/docs/upgrading.asciidoc b/libbeat/docs/upgrading.asciidoc index c4557d11071..466d99b9378 100644 --- a/libbeat/docs/upgrading.asciidoc +++ b/libbeat/docs/upgrading.asciidoc @@ -71,6 +71,11 @@ we've provided a migration tool to help you migrate your configurations from version 6.6 to 6.7 or later. For more information, see the https://www.elastic.co/blog/beats-6-7-0-released[Beats 6.7.0 release blog]. +NOTE: {beats} central management has been removed starting in version 7.14.0. +Looking for a replacement? Refer to the +{fleet-guide}/index.html[Fleet User Guide] to learn how to deploy and centrally +manage a single {agent} to monitor and secure each host. + ==== Upgrade {beats} binaries to 7.0 Before upgrading: diff --git a/libbeat/publisher/queue/diskqueue/core_loop_test.go b/libbeat/publisher/queue/diskqueue/core_loop_test.go index 3bf53e9b8a5..7c0d426ea61 100644 --- a/libbeat/publisher/queue/diskqueue/core_loop_test.go +++ b/libbeat/publisher/queue/diskqueue/core_loop_test.go @@ -235,7 +235,7 @@ func TestHandleWriterLoopResponse(t *testing.T) { // Write to one segment (no segments should be moved to reading list) dq.handleWriterLoopResponse(writerLoopResponse{ - segments: []writerLoopResponseSegment{ + segments: []writerLoopSegmentResponse{ {bytesWritten: 100}, }, }) @@ -250,7 +250,7 @@ func TestHandleWriterLoopResponse(t *testing.T) { // Write to two segments (the first one should be moved to reading list) dq.handleWriterLoopResponse(writerLoopResponse{ - segments: []writerLoopResponseSegment{ + segments: []writerLoopSegmentResponse{ {bytesWritten: 100}, {bytesWritten: 100}, }, @@ -270,7 +270,7 @@ func TestHandleWriterLoopResponse(t *testing.T) { // Write to three segments (the first two should be moved to reading list) dq.handleWriterLoopResponse(writerLoopResponse{ - segments: []writerLoopResponseSegment{ + segments: []writerLoopSegmentResponse{ {bytesWritten: 100}, {bytesWritten: 100}, {bytesWritten: 500}, diff --git a/libbeat/publisher/queue/diskqueue/reader_loop.go b/libbeat/publisher/queue/diskqueue/reader_loop.go index a31cc5b8a77..d6bb49494e0 100644 --- a/libbeat/publisher/queue/diskqueue/reader_loop.go +++ b/libbeat/publisher/queue/diskqueue/reader_loop.go @@ -20,6 +20,7 @@ package diskqueue import ( "encoding/binary" "fmt" + "io" "os" ) @@ -100,13 +101,12 @@ func (rl *readerLoop) processRequest(request readerLoopRequest) readerLoopRespon // Open the file and seek to the starting position. handle, err := request.segment.getReader(rl.settings) + rl.decoder.useJSON = request.segment.shouldUseJSON() if err != nil { return readerLoopResponse{err: err} } defer handle.Close() - // getReader positions us at the start of the data region, so we use - // a relative seek to advance to the request position. - _, err = handle.Seek(int64(request.startPosition), os.SEEK_CUR) + _, err = handle.Seek(int64(request.startPosition), io.SeekStart) if err != nil { return readerLoopResponse{err: err} } @@ -179,7 +179,7 @@ func (rl *readerLoop) nextFrame( // Ensure we are allowed to read the frame header. if maxLength < frameHeaderSize { return nil, fmt.Errorf( - "Can't read next frame: remaining length %d is too low", maxLength) + "can't read next frame: remaining length %d is too low", maxLength) } // Wrap the handle to retry non-fatal errors and always return the full // requested data length if possible. @@ -187,20 +187,20 @@ func (rl *readerLoop) nextFrame( var frameLength uint32 err := binary.Read(reader, binary.LittleEndian, &frameLength) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame header: %w", err) + return nil, fmt.Errorf("couldn't read data frame header: %w", err) } // If the frame extends past the area we were told to read, return an error. // This should never happen unless the segment file is corrupted. if maxLength < uint64(frameLength) { return nil, fmt.Errorf( - "Can't read next frame: frame size is %d but remaining data is only %d", + "can't read next frame: frame size is %d but remaining data is only %d", frameLength, maxLength) } if frameLength <= frameMetadataSize { // Valid enqueued data must have positive length return nil, fmt.Errorf( - "Data frame with no data (length %d)", frameLength) + "data frame with no data (length %d)", frameLength) } // Read the actual frame data @@ -208,29 +208,29 @@ func (rl *readerLoop) nextFrame( bytes := rl.decoder.Buffer(int(dataLength)) _, err = reader.Read(bytes) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame content: %w", err) + return nil, fmt.Errorf("couldn't read data frame content: %w", err) } // Read the footer (checksum + duplicate length) var checksum uint32 err = binary.Read(reader, binary.LittleEndian, &checksum) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame checksum: %w", err) + return nil, fmt.Errorf("couldn't read data frame checksum: %w", err) } expected := computeChecksum(bytes) if checksum != expected { return nil, fmt.Errorf( - "Data frame checksum mismatch (%x != %x)", checksum, expected) + "data frame checksum mismatch (%x != %x)", checksum, expected) } var duplicateLength uint32 err = binary.Read(reader, binary.LittleEndian, &duplicateLength) if err != nil { - return nil, fmt.Errorf("Couldn't read data frame footer: %w", err) + return nil, fmt.Errorf("couldn't read data frame footer: %w", err) } if duplicateLength != frameLength { return nil, fmt.Errorf( - "Inconsistent data frame length (%d vs %d)", + "inconsistent data frame length (%d vs %d)", frameLength, duplicateLength) } @@ -242,7 +242,7 @@ func (rl *readerLoop) nextFrame( // TODO: Rather than pass this error back to the read request, which // discards the rest of the segment, we should just log the error and // advance to the next frame, which is likely still valid. - return nil, fmt.Errorf("Couldn't decode data frame: %w", err) + return nil, fmt.Errorf("couldn't decode data frame: %w", err) } frame := &readFrame{ diff --git a/libbeat/publisher/queue/diskqueue/segments.go b/libbeat/publisher/queue/diskqueue/segments.go index f666a1941c8..450139b2d04 100644 --- a/libbeat/publisher/queue/diskqueue/segments.go +++ b/libbeat/publisher/queue/diskqueue/segments.go @@ -91,7 +91,10 @@ type queueSegment struct { // If this segment was loaded from a previous session, schemaVersion // points to the file schema version that was read from its header. // This is only used by queueSegment.headerSize(), which is used in - // maybeReadPending to calculate the position of the first data frame. + // maybeReadPending to calculate the position of the first data frame, + // and by queueSegment.shouldUseJSON(), which is used in the reader + // loop to detect old segments that used JSON encoding instead of + // the current CBOR. schemaVersion *uint32 // The number of bytes occupied by this segment on-disk, as of the most @@ -198,6 +201,14 @@ func (segment *queueSegment) headerSize() uint64 { return segmentHeaderSize } +// The initial release of the disk queue used JSON to encode events +// on disk. Since then, we have switched to CBOR to address issues +// with encoding multi-byte characters, and for lower encoding +// overhead. +func (segment *queueSegment) shouldUseJSON() bool { + return segment.schemaVersion != nil && *segment.schemaVersion == 0 +} + // Should only be called from the reader loop. If successful, returns an open // file handle positioned at the beginning of the segment's data region. func (segment *queueSegment) getReader( @@ -207,14 +218,14 @@ func (segment *queueSegment) getReader( file, err := os.Open(path) if err != nil { return nil, fmt.Errorf( - "Couldn't open segment %d: %w", segment.id, err) + "couldn't open segment %d: %w", segment.id, err) } // We don't need the header contents here, we just want to advance past the // header region, so discard the return value. _, err = readSegmentHeader(file) if err != nil { file.Close() - return nil, fmt.Errorf("Couldn't read segment header: %w", err) + return nil, fmt.Errorf("couldn't read segment header: %w", err) } return file, nil @@ -231,7 +242,7 @@ func (segment *queueSegment) getWriter( } err = writeSegmentHeader(file, 0) if err != nil { - return nil, fmt.Errorf("Couldn't write segment header: %w", err) + return nil, fmt.Errorf("couldn't write segment header: %w", err) } return file, nil @@ -306,7 +317,7 @@ func readSegmentHeaderWithFrameCount(path string) (*segmentHeader, error) { // the current frame to make sure the trailing length matches before // advancing to the next frame (otherwise we might accept an impossible // length). - _, err = file.Seek(int64(frameLength-8), os.SEEK_CUR) + _, err = file.Seek(int64(frameLength-8), io.SeekCurrent) if err != nil { break } @@ -341,7 +352,7 @@ func readSegmentHeader(in io.Reader) (*segmentHeader, error) { return nil, err } if header.version > currentSegmentVersion { - return nil, fmt.Errorf("Unrecognized schema version %d", header.version) + return nil, fmt.Errorf("unrecognized schema version %d", header.version) } if header.version >= 1 { err = binary.Read(in, binary.LittleEndian, &header.frameCount) diff --git a/libbeat/publisher/queue/diskqueue/serialize.go b/libbeat/publisher/queue/diskqueue/serialize.go index 456f180665b..fc0c24de673 100644 --- a/libbeat/publisher/queue/diskqueue/serialize.go +++ b/libbeat/publisher/queue/diskqueue/serialize.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/outputs/codec" "github.com/elastic/beats/v7/libbeat/publisher" + "github.com/elastic/go-structform/cborl" "github.com/elastic/go-structform/gotype" "github.com/elastic/go-structform/json" ) @@ -40,7 +41,13 @@ type eventEncoder struct { type eventDecoder struct { buf []byte - parser *json.Parser + jsonParser *json.Parser + cborlParser *cborl.Parser + + // Current serialization all uses CBOR. Set this flag when decoding + // from old (schema 0) segment files generated by the disk queue beta. + useJSON bool + unfolder *gotype.Unfolder } @@ -60,7 +67,7 @@ func newEventEncoder() *eventEncoder { func (e *eventEncoder) reset() { e.folder = nil - visitor := json.NewVisitor(&e.buf) + visitor := cborl.NewVisitor(&e.buf) // This can't return an error: NewIterator is deterministic based on its // input, and doesn't return an error when called with valid options. In // this case the options are hard-coded to fixed values, so they are @@ -109,7 +116,8 @@ func (d *eventDecoder) reset() { unfolder, _ := gotype.NewUnfolder(nil) d.unfolder = unfolder - d.parser = json.NewParser(unfolder) + d.jsonParser = json.NewParser(unfolder) + d.cborlParser = cborl.NewParser(unfolder) } // Buffer prepares the read buffer to hold the next event of n bytes. @@ -131,7 +139,11 @@ func (d *eventDecoder) Decode() (publisher.Event, error) { d.unfolder.SetTarget(&to) defer d.unfolder.Reset() - err = d.parser.Parse(d.buf) + if d.useJSON { + err = d.jsonParser.Parse(d.buf) + } else { + err = d.cborlParser.Parse(d.buf) + } if err != nil { d.reset() // reset parser just in case diff --git a/libbeat/publisher/queue/diskqueue/serialize_test.go b/libbeat/publisher/queue/diskqueue/serialize_test.go new file mode 100644 index 00000000000..888a42eca88 --- /dev/null +++ b/libbeat/publisher/queue/diskqueue/serialize_test.go @@ -0,0 +1,69 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package diskqueue + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/publisher" +) + +// A test to make sure serialization works correctly on multi-byte characters. +func TestSerialize(t *testing.T) { + testCases := []struct { + name string + value string + }{ + {name: "Ascii only", value: "{\"name\": \"Momotaro\"}"}, + {name: "Multi-byte", value: "{\"name\": \"桃太郎\"}"}, + } + + for _, test := range testCases { + encoder := newEventEncoder() + event := publisher.Event{ + Content: beat.Event{ + Fields: common.MapStr{ + "test_field": test.value, + }, + }, + } + serialized, err := encoder.encode(&event) + if err != nil { + t.Fatalf("[%v] Couldn't encode event: %v", test.name, err) + } + + // Use decoder to decode the serialized bytes. + decoder := newEventDecoder() + buf := decoder.Buffer(len(serialized)) + copy(buf, serialized) + decoded, err := decoder.Decode() + if err != nil { + t.Fatalf("[%v] Couldn't decode serialized data: %v", test.name, err) + } + + decodedValue, err := decoded.Content.Fields.GetValue("test_field") + if err != nil { + t.Fatalf("[%v] Couldn't get field 'test_field': %v", test.name, err) + } + assert.Equal(t, test.value, decodedValue) + } +} diff --git a/libbeat/publisher/queue/diskqueue/writer_loop.go b/libbeat/publisher/queue/diskqueue/writer_loop.go index ce7bed6b2b0..31a03f9d27e 100644 --- a/libbeat/publisher/queue/diskqueue/writer_loop.go +++ b/libbeat/publisher/queue/diskqueue/writer_loop.go @@ -49,9 +49,9 @@ type writerLoopRequest struct { frames []segmentedFrame } -// A writerLoopResponseSegment specifies the number of frames and bytes +// A writerLoopSegmentResponse specifies the number of frames and bytes // written to a single segment as a result of a writerLoopRequest. -type writerLoopResponseSegment struct { +type writerLoopSegmentResponse struct { framesWritten uint32 bytesWritten uint64 } @@ -61,7 +61,7 @@ type writerLoopResponseSegment struct { // segment that appeared in the request, in the same order. If there is // more than one entry, then all but the last segment have been closed. type writerLoopResponse struct { - segments []writerLoopResponseSegment + segments []writerLoopSegmentResponse } type writerLoop struct { @@ -143,7 +143,7 @@ func (wl *writerLoop) processRequest( // responseEntry tracks the number of frames and bytes written to the // current segment. - var curSegment writerLoopResponseSegment + var curSegmentResponse writerLoopSegmentResponse // response var response writerLoopResponse outerLoop: @@ -157,14 +157,14 @@ outerLoop: // Update the header with the frame count (including the ones we // just wrote), try to sync to disk, then close the file. writeSegmentHeader(wl.outputFile, - wl.currentSegment.frameCount+curSegment.framesWritten) + wl.currentSegment.frameCount+curSegmentResponse.framesWritten) wl.outputFile.Sync() wl.outputFile.Close() wl.outputFile = nil // We are done with this segment, add the totals to the response and // reset the current counters. - response.segments = append(response.segments, curSegment) - curSegment = writerLoopResponseSegment{} + response.segments = append(response.segments, curSegmentResponse) + curSegmentResponse = writerLoopSegmentResponse{} } wl.currentSegment = frameRequest.segment file, err := wl.currentSegment.getWriterWithRetry( @@ -173,6 +173,9 @@ outerLoop: // This can only happen if the queue is being closed; abort. break } + // We're creating a new segment file, set the initial bytes written + // to the header size. + curSegmentResponse.bytesWritten = wl.currentSegment.headerSize() wl.outputFile = file } // Make sure our writer points to the current file handle. @@ -208,8 +211,8 @@ outerLoop: // abort while a frame is partially written, we only report up to the // last complete frame. (This almost never matters, but it allows for // more controlled recovery after a bad shutdown.) - curSegment.framesWritten++ - curSegment.bytesWritten += uint64(frameSize) + curSegmentResponse.framesWritten++ + curSegmentResponse.bytesWritten += uint64(frameSize) // Update the ACKs that will be sent at the end of the request. totalACKCount++ @@ -238,7 +241,7 @@ outerLoop: } // Add the final segment to the response and return it. - response.segments = append(response.segments, curSegment) + response.segments = append(response.segments, curSegmentResponse) return response } diff --git a/metricbeat/docs/index.asciidoc b/metricbeat/docs/index.asciidoc index 8fff64854e8..d4801d75c34 100644 --- a/metricbeat/docs/index.asciidoc +++ b/metricbeat/docs/index.asciidoc @@ -17,7 +17,6 @@ include::{asciidoc-dir}/../../shared/attributes.asciidoc[] :discuss_forum: beats/{beatname_lc} :beat_default_index_prefix: {beatname_lc} :beat_kib_app: {kib} Metrics -:has_central_config: :has_solutions: :has_docker_label_ex: :has_modules_command: diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc index ba7686a7ccd..0eafcdf2036 100644 --- a/x-pack/elastic-agent/CHANGELOG.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.asciidoc @@ -44,6 +44,7 @@ - Fix windows installer during enroll {pull}24343[24343] - Logging to file disabled on enroll {issue}24173[24173] - Prevent uninstall failures on empty config {pull}24838[24838] +- Fix issue with FLEET_CA not being used with Fleet Server in container {pull}26529[26529] ==== New features diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 761ddd39ea2..ffd8febf29c 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -116,3 +116,5 @@ - Enable agent to send custom headers to kibana/ES {pull}26275[26275] - Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394] - Add proxy support to artifact downloader and communication with fleet server. {pull}25219[25219] +- Enable configuring monitoring namespace {issue}26439[26439] +- Communicate with Fleet Server over HTTP2. {pull}26474[26474] diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go index 8c3eb1c7d43..b31220f93de 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go @@ -49,6 +49,7 @@ func InjectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr transpiler.NewKey("logs", transpiler.NewBoolVal(true)), transpiler.NewKey("metrics", transpiler.NewBoolVal(true)), transpiler.NewKey("use_output", transpiler.NewStrVal("default")), + transpiler.NewKey("namespace", transpiler.NewStrVal("default")), }) transpiler.Insert(rootAst, transpiler.NewKey("monitoring", monitoringNode), "settings") diff --git a/x-pack/elastic-agent/pkg/agent/cmd/container.go b/x-pack/elastic-agent/pkg/agent/cmd/container.go index 8f8b5b4ab8a..41041656518 100644 --- a/x-pack/elastic-agent/pkg/agent/cmd/container.go +++ b/x-pack/elastic-agent/pkg/agent/cmd/container.go @@ -386,9 +386,9 @@ func buildEnrollArgs(cfg setupConfig, token string, policyID string) ([]string, if cfg.Fleet.Insecure { args = append(args, "--insecure") } - if cfg.Fleet.CA != "" { - args = append(args, "--certificate-authorities", cfg.Fleet.CA) - } + } + if cfg.Fleet.CA != "" { + args = append(args, "--certificate-authorities", cfg.Fleet.CA) } if token != "" { args = append(args, "--enrollment-token", token) diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 45b7263cf73..17188321b56 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -161,11 +161,12 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in var steps []configrequest.Step watchLogs := o.monitor.WatchLogs() watchMetrics := o.monitor.WatchMetrics() + monitoringNamespace := o.monitor.MonitoringNamespace() // generate only when monitoring is running (for config refresh) or // state changes (turning on/off) if watchLogs != o.isMonitoringLogs() || watchLogs { - fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output) + fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output, monitoringNamespace) stepID := configrequest.StepRun if !watchLogs || !any { stepID = configrequest.StepRemove @@ -182,7 +183,7 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in steps = append(steps, filebeatStep) } if watchMetrics != o.isMonitoringMetrics() || watchMetrics { - mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output) + mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output, monitoringNamespace) stepID := configrequest.StepRun if !watchMetrics || !any { stepID = configrequest.StepRemove @@ -215,12 +216,12 @@ func loadSpecFromSupported(processName string) program.Spec { } } -func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { inputs := []interface{}{ map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -233,7 +234,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"), filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"), }, - "index": "logs-elastic_agent-default", + "index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -241,7 +242,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa "fields": map[string]interface{}{ "type": "logs", "dataset": "elastic_agent", - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -280,7 +281,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa inputs = append(inputs, map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -288,7 +289,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa }, }, "paths": paths, - "index": fmt.Sprintf("logs-elastic_agent.%s-default", name), + "index": fmt.Sprintf("logs-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -296,7 +297,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa "fields": map[string]interface{}{ "type": "logs", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -345,7 +346,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa return result, true } -func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { hosts := o.getMetricbeatEndpoints() if len(hosts) == 0 { return nil, false @@ -359,7 +360,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "metricsets": []string{"stats", "state"}, "period": "10s", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", name), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -367,7 +368,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -397,7 +398,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "period": "10s", "path": "/stats", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -405,7 +406,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -480,7 +481,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "period": "10s", "path": "/stats", "hosts": []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS(), o.config.MonitoringConfig.HTTP)}, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -488,7 +489,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go index 136c9e485b1..c23248ff2d9 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go @@ -215,6 +215,9 @@ func (b *testMonitor) Reload(cfg *config.Config) error { return nil } // IsMonitoringEnabled returns true if monitoring is configured. func (b *testMonitor) IsMonitoringEnabled() bool { return b.monitorLogs || b.monitorMetrics } +// MonitoringNamespace returns monitoring namespace configured. +func (b *testMonitor) MonitoringNamespace() string { return "default" } + // WatchLogs return true if monitoring is configured and monitoring logs is enabled. func (b *testMonitor) WatchLogs() bool { return b.monitorLogs } diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index 4498f7e5236..5ca35de0136 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -383,6 +383,10 @@ func TestConfiguration(t *testing.T) { empty bool err bool }{ + "namespace": { + programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, + expected: 6, + }, "single_config": { programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, expected: 6, diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml new file mode 100644 index 00000000000..7e9f04dc411 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml @@ -0,0 +1,114 @@ +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: + - "127.0.0.1:9200" + - "127.0.0.1:9300" + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + +inputs: +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml new file mode 100644 index 00000000000..83df83e56e0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml @@ -0,0 +1,68 @@ +filebeat: + inputs: + - type: log + paths: + - /var/log/hello1.log + - /var/log/hello2.log + index: logs-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: logs + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - type: log + paths: + - /var/log/hello3.log + - /var/log/hello4.log + index: testtype-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: testtype + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml new file mode 100644 index 00000000000..c03696aff1f --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml @@ -0,0 +1,16 @@ +fleet: + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +inputs: + - id: fleet-server-id + type: fleet-server diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml new file mode 100644 index 00000000000..f34b204f5fa --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml @@ -0,0 +1,30 @@ +inputs: +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s + data_stream.namespace: default + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml new file mode 100644 index 00000000000..3f16a9d9e21 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml @@ -0,0 +1,88 @@ +metricbeat: + modules: + - module: docker + metricsets: [status] + index: metrics-docker.status-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: docker.status + namespace: default + - add_fields: + target: "event" + fields: + dataset: docker.status + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: docker + metricsets: [info] + index: metrics-generic-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: apache + metricsets: [info] + index: metrics-generic-testing + hosts: ["http://apache.remote"] + processors: + - add_fields: + fields: + should_be: first + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: testing + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml new file mode 100644 index 00000000000..d71499bdef4 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml @@ -0,0 +1,35 @@ +inputs: +- type: packet + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml new file mode 100644 index 00000000000..c2f83a9abf0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml @@ -0,0 +1,201 @@ +name: Production Website DB Servers +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + server: + output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +outputs: + default: + type: elasticsearch + namespace: test_namespace + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + + monitoring: + type: elasticsearch + api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + hosts: ["monitoring:9200"] + ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M=" + +inputs: +- id: fleet-server-id + type: fleet-server + use_output: default + data_stream: + type: default +- type: docker/metrics + use_output: default + streams: + - metricset: status + processors: + - null + data_stream: + dataset: docker.status + - metricset: info + data_stream: + dataset: "" + hosts: ["http://127.0.0.1:8080"] +- type: logfile + use_output: default + streams: + - paths: + - /var/log/hello1.log + - /var/log/hello2.log + vars: + var: value +- type: logfile + data_stream: + type: testtype + use_output: default + streams: + - paths: + - /var/log/hello3.log + - /var/log/hello4.log + vars: + var: value +- id: apache-metrics-id + type: apache/metrics + data_stream: + namespace: testing + use_output: default + processors: + - add_fields: + fields: + should_be: first + streams: + - enabled: true + metricset: info + hosts: ["http://apache.remote"] + hosts: ["http://apache.local"] +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s +- type: packet + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + +agent.monitoring: + use_output: monitoring + +agent: + reload: 123 diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go index 100d8a462a0..31bb2faaa7c 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go @@ -716,8 +716,10 @@ func load(val reflect.Value) (Node, error) { return loadSliceOrArray(val) case reflect.String: return &StrVal{value: val.Interface().(string)}, nil - case reflect.Int, reflect.Int64: + case reflect.Int: return &IntVal{value: val.Interface().(int)}, nil + case reflect.Int64: + return &IntVal{value: int(val.Interface().(int64))}, nil case reflect.Uint: return &UIntVal{value: uint64(val.Interface().(uint))}, nil case reflect.Uint64: diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go index e1b22c390ed..1e52d6c6f47 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go @@ -105,6 +105,7 @@ func TestAST(t *testing.T) { "support integers": { hashmap: map[string]interface{}{ "timeout": 12, + "zero": int64(0), "range": []int{20, 30, 40}, }, ast: &AST{ @@ -121,6 +122,7 @@ func TestAST(t *testing.T) { ), }, &Key{name: "timeout", value: &IntVal{value: 12}}, + &Key{name: "zero", value: &IntVal{value: 0}}, }, }, }, diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index 743d44118d6..1c0c4ba61ad 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -21,6 +21,7 @@ import ( ) const httpPlusPrefix = "http+" +const defaultMonitoringNamespace = "default" // Monitor is a monitoring interface providing information about the way // how beat is monitored @@ -69,6 +70,14 @@ func (b *Monitor) Close() { // IsMonitoringEnabled returns true if monitoring is enabled. func (b *Monitor) IsMonitoringEnabled() bool { return b.config.Enabled } +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { + if b.config.Namespace == "" { + return defaultMonitoringNamespace + } + return b.config.Namespace +} + // WatchLogs returns true if monitoring is enabled and monitor should watch logs. func (b *Monitor) WatchLogs() bool { return b.config.Enabled && b.config.MonitorLogs } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go index 2ce067d4e19..fe18b0fb73e 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go @@ -5,6 +5,7 @@ package config const defaultPort = 6791 +const defaultNamespace = "default" // MonitoringConfig describes a configuration of a monitoring type MonitoringConfig struct { @@ -12,6 +13,7 @@ type MonitoringConfig struct { MonitorLogs bool `yaml:"logs" config:"logs"` MonitorMetrics bool `yaml:"metrics" config:"metrics"` HTTP *MonitoringHTTPConfig `yaml:"http" config:"http"` + Namespace string `yaml:"namespace" config:"namespace"` } // MonitoringHTTPConfig is a config defining HTTP endpoint published by agent @@ -33,5 +35,6 @@ func DefaultConfig() *MonitoringConfig { Enabled: false, Port: defaultPort, }, + Namespace: defaultNamespace, } } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go index 00c7a50003a..6c71f4f65fc 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go @@ -23,6 +23,7 @@ type Monitor interface { Cleanup(spec program.Spec, pipelineID string) error Reload(cfg *config.Config) error IsMonitoringEnabled() bool + MonitoringNamespace() string WatchLogs() bool WatchMetrics() bool Close() diff --git a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go index 9ea8f08a788..d98deb90888 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go @@ -66,3 +66,6 @@ func (b *Monitor) WatchLogs() bool { return false } // WatchMetrics return true if monitoring is configured and monitoring metrics is enabled. func (b *Monitor) WatchMetrics() bool { return false } + +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { return "default" } diff --git a/x-pack/elastic-agent/pkg/remote/client.go b/x-pack/elastic-agent/pkg/remote/client.go index 34db0bcaf2d..996f0aabd81 100644 --- a/x-pack/elastic-agent/pkg/remote/client.go +++ b/x-pack/elastic-agent/pkg/remote/client.go @@ -112,33 +112,40 @@ func NewWithConfig(log *logger.Logger, cfg Config, wrapper wrapperFunc) (*Client hosts := cfg.GetHosts() clients := make([]*requestClient, len(hosts)) for i, host := range cfg.GetHosts() { - var errWrap error - httpClient, err := cfg.Transport.Client( - httpcommon.WithAPMHTTPInstrumentation(), - httpcommon.WithModRoundtripper(func(rt http.RoundTripper) http.RoundTripper { - if cfg.IsBasicAuth() { - // Pass basic auth credentials to all the underlying calls. - rt = NewBasicAuthRoundTripper(rt, cfg.Username, cfg.Password) - } - if wrapper != nil { - rt, errWrap = wrapper(rt) - } - return rt - }), - ) + connStr, err := common.MakeURL(string(cfg.Protocol), p, host, 0) if err != nil { - err = errors.Wrap(err, "fail to create transport client") - } else if errWrap != nil { - err = errors.Wrap(errWrap, "fail to create transport client") + return nil, errors.Wrap(err, "invalid fleet-server endpoint") } - - url, err := common.MakeURL(string(cfg.Protocol), p, host, 0) + addr, err := url.Parse(connStr) if err != nil { return nil, errors.Wrap(err, "invalid fleet-server endpoint") } + + transport, err := makeTransport(cfg, addr.Scheme) + if err != nil { + return nil, err + } + + if cfg.IsBasicAuth() { + // Pass basic auth credentials to all the underlying calls. + transport = NewBasicAuthRoundTripper(transport, cfg.Username, cfg.Password) + } + + if wrapper != nil { + transport, err = wrapper(transport) + if err != nil { + return nil, errors.Wrap(err, "fail to create transport client") + } + } + + httpClient := http.Client{ + Transport: transport, + Timeout: cfg.Transport.Timeout, + } + clients[i] = &requestClient{ - request: prefixRequestFactory(url), - client: *httpClient, + request: prefixRequestFactory(connStr), + client: httpClient, } } @@ -266,3 +273,22 @@ func prefixRequestFactory(URL string) requestFunc { return http.NewRequest(method, newPath, body) } } + +// makeTransport create a transport object based on the TLS configuration. +func makeTransport(cfg Config, scheme string) (http.RoundTripper, error) { + opts := []httpcommon.TransportOption{ + httpcommon.WithAPMHTTPInstrumentation(), + } + + // Connect to fleet server via HTTP2 only if no proxy is configured. + // The HTTP2 only transport will ignore HTTP_PROXY, HTTPS_PPROXY, and NO_PROXY + // environment variables. + http2Only := scheme == "https" + if http2Only && cfg.Transport.Proxy.URL == nil { + opts = append(opts, httpcommon.WithHTTP2Only(true)) + } else { + opts = append(opts, httpcommon.WithForceAttemptHTTP2(true)) + } + + return cfg.Transport.RoundTripper(opts...) +} diff --git a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl index 8b2b53fa8c7..13cb37c0f9c 100644 --- a/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl +++ b/x-pack/filebeat/_meta/config/filebeat.inputs.reference.xpack.yml.tmpl @@ -76,3 +76,60 @@ # List of S3 object metadata keys to include in events. #include_s3_metadata: [] + +#------------------------------ AWS CloudWatch input -------------------------------- +# Beta: Config options for AWS CloudWatch input +#- type: aws-cloudwatch + #enabled: false + + # AWS Credentials + # If access_key_id and secret_access_key are configured, then use them to make api calls. + # If not, aws-cloudwatch input will load default AWS config or load with given profile name. + #access_key_id: '${AWS_ACCESS_KEY_ID:""}' + #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + #session_token: '${AWS_SESSION_TOKEN:"”}' + #credential_profile_name: test-aws-s3-input + + # ARN of the log group to collect logs from + #log_group_arn: "arn:aws:logs:us-east-1:428152502467:log-group:test:*" + + # Name of the log group to collect logs from. + # Note: region_name is required when log_group_name is given. + #log_group_name: test + + # The prefix for a group of log group names. + # Note: `region_name` is required when `log_group_name_prefix` is given. + # `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + #log_group_name_prefix: /aws/ + + # Region that the specified log group or log group prefix belongs to. + #region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from. + #log_streams: + # - log_stream_name + + # A string to filter the results to include only log events from log streams + # that have names starting with this prefix. + #log_stream_prefix: test + + # `start_position` allows user to specify if this input should read log files + # from the `beginning` or from the `end`. + # `beginning`: reads from the beginning of the log group (default). + # `end`: read only new messages from current time minus `scan_frequency` going forward. + #start_position: beginning + + # This config parameter sets how often Filebeat checks for new log events from the + # specified log group. Default `scan_frequency` is 1 minute, which means Filebeat + # will sleep for 1 minute before querying for new logs again. + #scan_frequency: 1m + + # The maximum duration of AWS API can take. If it exceeds the timeout, AWS API + # will be interrupted. + # The default AWS API timeout for a message is 120 seconds. + # The minimum is 0 seconds. + #api_timeout: 120s + + # This is used to sleep between AWS `FilterLogEvents` API calls inside the same + # collection period. + #api_sleep: 200ms diff --git a/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc b/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc index d39ec550868..e04eaa8b9e2 100644 --- a/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc @@ -45,12 +45,18 @@ ARN of the log group to collect logs from. [float] ==== `log_group_name` -Name of the log group to collect logs from. Note: region_name is required when +Name of the log group to collect logs from. Note: `region_name` is required when log_group_name is given. +[float] +==== `log_group_name_prefix` +The prefix for a group of log group names. Note: `region_name` is required when +log_group_name_prefix is given. `log_group_name` and `log_group_name_prefix` +cannot be given at the same time. + [float] ==== `region_name` -Region that the specified log group belongs to. +Region that the specified log group or log group prefix belongs to. [float] ==== `log_streams` @@ -109,6 +115,7 @@ Please see <> for more details. === AWS Permissions Specific AWS permissions are required for IAM user to access aws-cloudwatch: ---- +cloudwatchlogs:DescribeLogGroups logs:FilterLogEvents ---- diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index f55994e9c83..9a797d337fc 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -2274,6 +2274,38 @@ filebeat.modules: # var.ssl_certificate: path/to/server_ssl_cert.pem # var.ssl_key: path/to/ssl_key.pem + recordedfuture: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The interval to poll the API for updates + var.interval: 5m + + # How far back in time to start fetching intelligence when run for the + # first time. Value must be in hours. Default: 168h (1 week). + var.first_interval: 168h + + # The URL used for Threat Intel API calls. + # Must include the `limit` parameter and at least `entity` and `timestamps` fields. + # See the Connect API Explorer for a list of possible parameters. + # + # For `ip` entities: + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `domain` entities: + # var.url: "https://api.recordedfuture.com/v2/domain/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `hash` entities: + # var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + + # For `url` entities: + # var.url: "https://api.recordedfuture.com/v2/url/search?limit=200&fields=entity,timestamps,risk&metadata=false" + + # Set your API Token. + var.api_token: "" + #---------------------------- Apache Tomcat Module ---------------------------- - module: tomcat log: @@ -3095,6 +3127,63 @@ filebeat.inputs: # List of S3 object metadata keys to include in events. #include_s3_metadata: [] +#------------------------------ AWS CloudWatch input -------------------------------- +# Beta: Config options for AWS CloudWatch input +#- type: aws-cloudwatch + #enabled: false + + # AWS Credentials + # If access_key_id and secret_access_key are configured, then use them to make api calls. + # If not, aws-cloudwatch input will load default AWS config or load with given profile name. + #access_key_id: '${AWS_ACCESS_KEY_ID:""}' + #secret_access_key: '${AWS_SECRET_ACCESS_KEY:""}' + #session_token: '${AWS_SESSION_TOKEN:"”}' + #credential_profile_name: test-aws-s3-input + + # ARN of the log group to collect logs from + #log_group_arn: "arn:aws:logs:us-east-1:428152502467:log-group:test:*" + + # Name of the log group to collect logs from. + # Note: region_name is required when log_group_name is given. + #log_group_name: test + + # The prefix for a group of log group names. + # Note: `region_name` is required when `log_group_name_prefix` is given. + # `log_group_name` and `log_group_name_prefix` cannot be given at the same time. + #log_group_name_prefix: /aws/ + + # Region that the specified log group or log group prefix belongs to. + #region_name: us-east-1 + + # A list of strings of log streams names that Filebeat collect log events from. + #log_streams: + # - log_stream_name + + # A string to filter the results to include only log events from log streams + # that have names starting with this prefix. + #log_stream_prefix: test + + # `start_position` allows user to specify if this input should read log files + # from the `beginning` or from the `end`. + # `beginning`: reads from the beginning of the log group (default). + # `end`: read only new messages from current time minus `scan_frequency` going forward. + #start_position: beginning + + # This config parameter sets how often Filebeat checks for new log events from the + # specified log group. Default `scan_frequency` is 1 minute, which means Filebeat + # will sleep for 1 minute before querying for new logs again. + #scan_frequency: 1m + + # The maximum duration of AWS API can take. If it exceeds the timeout, AWS API + # will be interrupted. + # The default AWS API timeout for a message is 120 seconds. + # The minimum is 0 seconds. + #api_timeout: 120s + + # This is used to sleep between AWS `FilterLogEvents` API calls inside the same + # collection period. + #api_sleep: 200ms + # =========================== Filebeat autodiscover ============================ # Autodiscover allows you to detect changes in the system and spawn new modules diff --git a/x-pack/filebeat/input/awscloudwatch/config.go b/x-pack/filebeat/input/awscloudwatch/config.go index 7f1e4ca3ccd..210e7bb9a31 100644 --- a/x-pack/filebeat/input/awscloudwatch/config.go +++ b/x-pack/filebeat/input/awscloudwatch/config.go @@ -16,6 +16,7 @@ type config struct { harvester.ForwarderConfig `config:",inline"` LogGroupARN string `config:"log_group_arn"` LogGroupName string `config:"log_group_name"` + LogGroupNamePrefix string `config:"log_group_name_prefix"` RegionName string `config:"region_name"` LogStreams []string `config:"log_streams"` LogStreamPrefix string `config:"log_stream_prefix"` @@ -44,13 +45,17 @@ func (c *config) Validate() error { "either 'beginning' or 'end'") } - if c.LogGroupARN == "" && c.LogGroupName == "" { - return errors.New("log_group_arn and log_group_name config parameter" + - "cannot be both empty") + if c.LogGroupARN == "" && c.LogGroupName == "" && c.LogGroupNamePrefix == "" { + return errors.New("log_group_arn, log_group_name and log_group_name_prefix config parameter" + + "cannot all be empty") } - if c.LogGroupName != "" && c.RegionName == "" { - return errors.New("region_name is required when log_group_name " + + if c.LogGroupName != "" && c.LogGroupNamePrefix != "" { + return errors.New("log_group_name and log_group_name_prefix cannot be given at the same time") + } + + if (c.LogGroupName != "" || c.LogGroupNamePrefix != "") && c.RegionName == "" { + return errors.New("region_name is required when log_group_name or log_group_name_prefix " + "config parameter is given") } return nil diff --git a/x-pack/filebeat/input/awscloudwatch/input.go b/x-pack/filebeat/input/awscloudwatch/input.go index 2f3563a369e..7eb14147d31 100644 --- a/x-pack/filebeat/input/awscloudwatch/input.go +++ b/x-pack/filebeat/input/awscloudwatch/input.go @@ -131,15 +131,33 @@ func NewInput(cfg *common.Config, connector channel.Connector, context input.Con // Run runs the input func (in *awsCloudWatchInput) Run() { - in.workerOnce.Do(func() { - in.workerWg.Add(1) - go func() { - in.logger.Infof("aws-cloudwatch input worker for log group: '%v' has started", in.config.LogGroupName) - defer in.logger.Infof("aws-cloudwatch input worker for log group '%v' has stopped.", in.config.LogGroupName) - defer in.workerWg.Done() - in.run() - }() - }) + cwConfig := awscommon.EnrichAWSConfigWithEndpoint(in.config.AwsConfig.Endpoint, "cloudwatchlogs", in.config.RegionName, in.awsConfig) + svc := cloudwatchlogs.New(cwConfig) + + var logGroupNames []string + var err error + if in.config.LogGroupNamePrefix != "" { + logGroupNames, err = in.getLogGroupNames(svc) + if err != nil { + in.logger.Error("getLogGroupNames failed: ", err) + return + } + } else { + logGroupNames = []string{in.config.LogGroupName} + } + + for _, logGroup := range logGroupNames { + in.config.LogGroupName = logGroup + in.workerOnce.Do(func() { + in.workerWg.Add(1) + go func() { + in.logger.Infof("aws-cloudwatch input worker for log group: '%v' has started", in.config.LogGroupName) + defer in.logger.Infof("aws-cloudwatch input worker for log group '%v' has stopped.", in.config.LogGroupName) + defer in.workerWg.Done() + in.run() + }() + }) + } } func (in *awsCloudWatchInput) run() { @@ -176,6 +194,32 @@ func parseARN(logGroupARN string) (string, string, error) { return "", "", errors.Errorf("cannot get log group name from log group ARN: %s", logGroupARN) } +// getLogGroupNames uses DescribeLogGroups API to retrieve all log group names +func (in *awsCloudWatchInput) getLogGroupNames(svc cloudwatchlogsiface.ClientAPI) ([]string, error) { + // construct DescribeLogGroupsInput + filterLogEventsInput := &cloudwatchlogs.DescribeLogGroupsInput{ + LogGroupNamePrefix: awssdk.String(in.config.LogGroupNamePrefix), + } + + // make API request + req := svc.DescribeLogGroupsRequest(filterLogEventsInput) + p := cloudwatchlogs.NewDescribeLogGroupsPaginator(req) + var logGroupNames []string + for p.Next(context.TODO()) { + page := p.CurrentPage() + in.logger.Debugf("Collecting #%v log group names", len(page.LogGroups)) + for _, lg := range page.LogGroups { + logGroupNames = append(logGroupNames, *lg.LogGroupName) + } + } + + if err := p.Err(); err != nil { + in.logger.Error("failed DescribeLogGroupsRequest: ", err) + return logGroupNames, err + } + return logGroupNames, nil +} + // getLogEventsFromCloudWatch uses FilterLogEvents API to collect logs from CloudWatch func (in *awsCloudWatchInput) getLogEventsFromCloudWatch(svc cloudwatchlogsiface.ClientAPI) error { ctx, cancelFn := context.WithTimeout(in.inputCtx, in.config.APITimeout) diff --git a/x-pack/filebeat/module/threatintel/_meta/config.yml b/x-pack/filebeat/module/threatintel/_meta/config.yml index ce5b5271472..f2cf00bcf0d 100644 --- a/x-pack/filebeat/module/threatintel/_meta/config.yml +++ b/x-pack/filebeat/module/threatintel/_meta/config.yml @@ -137,3 +137,35 @@ # # var.ssl_certificate: path/to/server_ssl_cert.pem # var.ssl_key: path/to/ssl_key.pem + + recordedfuture: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The interval to poll the API for updates + var.interval: 5m + + # How far back in time to start fetching intelligence when run for the + # first time. Value must be in hours. Default: 168h (1 week). + var.first_interval: 168h + + # The URL used for Threat Intel API calls. + # Must include the `limit` parameter and at least `entity` and `timestamps` fields. + # See the Connect API Explorer for a list of possible parameters. + # + # For `ip` entities: + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `domain` entities: + # var.url: "https://api.recordedfuture.com/v2/domain/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `hash` entities: + # var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + + # For `url` entities: + # var.url: "https://api.recordedfuture.com/v2/url/search?limit=200&fields=entity,timestamps,risk&metadata=false" + + # Set your API Token. + var.api_token: "" diff --git a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc index 177f5264601..3c23bdc287e 100644 --- a/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/threatintel/_meta/docs.asciidoc @@ -24,6 +24,7 @@ The available filesets are: * <>: Supports gathering threat intel attributes from AlientVault OTX. * <>: Supports gathering threat intel attributes from Anomali Limo. * <>: Supports gathering threat intel attributes from Anomali ThreatStream. +* <>: Supports gathering threat intel attributes from Recorded Future. include::../include/gs-link.asciidoc[] @@ -218,7 +219,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -292,7 +293,7 @@ How often the API is polled for updated information. *`var.first_interval`*:: -How far back to search when retrieving events the first time the beat starts up. +How far back to search when retrieving events the first time the {beatname_uc} starts up. After the first interval has passed the module itself will use the timestamp from the last response as the filter when retrieving new events. @@ -404,7 +405,7 @@ Anomali Threat Intel is mapped to the following ECS fields. To configure the ThreatStream integration you first need to define an output in the Anomali ThreatStream Integrator using the Elastic SDK provided by Anomali. -It will deliver indicators via HTTP or HTTPS to a Filebeat instance running as +It will deliver indicators via HTTP or HTTPS to a {beatname_uc} instance running as a server. Configure an Integrator output with the following settings: @@ -414,12 +415,12 @@ Configure an Integrator output with the following settings: Adjust the paths to the python executable and the directory where the Elastic SDK has been unpacked. * Metadata in JSON Format: `{"url": "https://filebeat:8080/", "server_certificate": "/path/to/cert.pem", "secret": "my secret"}`. - - `url`: Use the host and port where Filebeat will be running, and `http` or `https` accordingly. + - `url`: Use the host and port where {beatname_uc} will be running, and `http` or `https` accordingly. - `server_certificate`: If using HTTPS, absolute path to the server certificate. Otherwise don't set this field. - - `secret`: A shared secret string to authenticate messages between the SDK and Filebeat. + - `secret`: A shared secret string to authenticate messages between the SDK and {beatname_uc}. -Then configure the `anomalithreatstream` fileset in Filebeat accordingly: +Then configure the `anomalithreatstream` fileset in {beatname_uc} accordingly: [source,yaml] ---- - module: threatintel @@ -444,11 +445,11 @@ Port number to use for the HTTP server. *`var.secret`*:: -Shared secret between the SDK and Filebeat, used to authenticate messages. +Shared secret between the SDK and {beatname_uc}, used to authenticate messages. *`var.ssl_certificate`*:: -Path to the public SSL certificate for the HTTPS server. If unset, Filebeat +Path to the public SSL certificate for the HTTPS server. If unset, {beatname_uc} will use unsecure HTTP connections. *`var.ssl_key`*:: @@ -483,6 +484,94 @@ Anomali ThreatStream fields are mapped to the following ECS fields: [[a]] [small]#[1]: Field is used to derive a value for the ECS field but its original value is kept under `threatintel.anomalithreatstream`.# +[[recordedfuture]] +[float] +==== `recordedfuture` fileset settings + +The `recordedfuture` fileset fetches intelligence from the Recorded Future Connect API. +It supports `domain`, `hash`, `ip` and `url` data types. + +To enable it you need to define the URL to fetch data from. You can construct this URL +using the https://api.recordedfuture.com/index.html[Recorded Future API Explorer.] The URL +must point to the `/search` endpoint and contain a suitable `limit` +(how many records to return from a single request) and `fields` parameters. +The `entity` and `timestamps` fields are required. + +Sample configuration: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +To fetch threat intelligence from multiple data types, you must define more than +one instance of the module: +[source,yaml] +---- +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 5m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +- module: threatintel + recordedfuture: + enabled: true + var.input: httpjson + var.interval: 1m + var.first_interval: 168h + var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + var.api_token: "" +---- + +*`var.url`*:: + +The URL of the API endpoint to connect with. + +*`var.api_token`*:: + +The API token used to access Recorded Future API. + +*`var.interval`*:: + +How often the API is polled for updated information. + +*`var.first_interval`*:: + +How far back to search when retrieving events the first time {beatname_uc} starts up. +After the first interval has passed the module itself will use the timestamp +from the last response as the filter when retrieving new events. + +*`var.proxy_url`*:: + +Optional URL to use as HTTP proxy. + + +Recorded Future fields are mapped to the following ECS fields: + +[options="header"] +|============================================================= +| Recorded Future fields | ECS Fields +| entity.name | threatintel.indicator.{url,ip,domain,file.hash} +| entity.type | threatintel.indicator.type +| fileHashes | threatintel.indicator.file.hash +| intelCard | event.reference +| location.asn | threatintel.indicator.as.number +| location.location | threatintel.indicator.geo +| location.organization | threatintel.indicator.as.organization.name +| risk.score | event.risk_score +| timestamps.firstSeen | threatintel.indicator.first_seen +| timestamps.lastSeen | threatintel.indicator.last_seen +|============================================================= + :has-dashboards!: [float] diff --git a/x-pack/filebeat/module/threatintel/_meta/fields.yml b/x-pack/filebeat/module/threatintel/_meta/fields.yml index 48222e31ebd..fce6811f5fe 100644 --- a/x-pack/filebeat/module/threatintel/_meta/fields.yml +++ b/x-pack/filebeat/module/threatintel/_meta/fields.yml @@ -167,6 +167,11 @@ ignore_above: 1024 description: City name. example: Montreal + - name: continent_name + type: keyword + ignore_above: 1024 + description: Name of the continent. + example: North America - name: country_iso_code type: keyword ignore_above: 1024 diff --git a/x-pack/filebeat/module/threatintel/fields.go b/x-pack/filebeat/module/threatintel/fields.go index de37293b142..a0b33b519a2 100644 --- a/x-pack/filebeat/module/threatintel/fields.go +++ b/x-pack/filebeat/module/threatintel/fields.go @@ -19,5 +19,5 @@ func init() { // AssetThreatintel returns asset data. // This is the base64 encoded gzipped contents of module/threatintel. func AssetThreatintel() string { - return "eJzsfW1z2ziS//v5FF3+v4i9JSm2Y2cnrsr/ymM7E9U5TtYPM1t7uVIgsiViDQIMAErWXt13v8IDKT6Jki0qN7e1fpNIBLt/aDQaje4G1IdHXJyBjiQSTblG9hOAppph9UuJDInCMxijJj8BhKgCSRNNBT+D//8TAMC9fQHsG4xOkQcIHyjDsfn2kwhThoOfACYUWajO7Ct94CSu8TJ/IU5IyvTItj6DCWEK/SO9SPAMplKkSd64Bsb8fbCcYCJFDDrCIpclsDgHZv6K4IoAKQ9pQLSQgwmVSo8UIs8bZZAecTEXMix8vwKYkxZCSDQC4SFoGiPMI+Rl6SmRygDBsgSJiZAaQ1B0GmnKp6AjqgrIWkAzsgqzgdA5YMNuK7zZK6qGlwk+3QzvTRqPUYKYWLCqwh3mRIEYK5QzDCEQPEwDD9JqMQk0nVG9aENpEG2pBIsEDcIlLKKM4CQq5EZ24wVcLEw3PluoZMwQKIe7++Ff4XhwOChRu3pKMDBvzQhLUZWeAfwJSKoFF7FIVV8tlMa43kJqOiGBrj0IqcRAC7moPxExobxvZFN7hjGhrE/CUNYeTSirt6fJ7KS5OU1mb5ufxCRY8SDV+FT7NpEiQFWXjRITPSeyjimVrP6dQtknQSBSXhfVnPJQzFVf4pQqLRf9R6xL7al/eviuH6CRtxl5bFGzgv5sp22X9tHYzgW0BIzuGU0XPJsCTueW1rJ1lgaEc5QjpYneZqZeGDkaKOe/vb66vIUZ8lBIg5JoUGlgBmySMraAELXT8JgwGlCRKqtIICQ83F63YU2kmNEQ5XYSHIbIzYh5ERom1sBEWLaDGbc2RIHgExqa5p1iWpIFSaxFI0rRKV8ObAYOUmUeW1tSeEsFhKF6nmW5ERruEgwMjLAHN4JjD67FvAefMKRp3IOPdBrVXjvsHx3WvjwPYyoJo3oBdwYK7B/13x7Uml3eDLPnp/13p/UGv199yRoM40QoRY3x7MMFSk0oP2gZGucT7ERVlJNS4P0OtyqFRBOgCgIRmxExDkub5pjmCvVu8XkmS++pYaFvA0mTGr7SV5tBI9l6XFohCYfhFzBWH5WCfSolGtyazmwf3GJFBT9olaJduDoT4gqkfn18KUjjQm1hWTfAZzgAd77SC0HaZX7gh2PXAuXOq9h29GMiHymfDjSr6+nzXDhJJma+XBuXFb5IoUUgGKiISDOXPZ92ewpE4lndgEVU1/2RX2XZjXffnpvxq317i2GrCHQQYVjrfnFnBQ17oiUtokVMgxLbVRJslWHNGjnChZG3roAHDASYCAgD5DMqBY+Ra0AeJoIaN0ICRz0X8hFwhlwPGnDbHu0AtqWbOQQ77kNl89FVFzKvcGc9qOshqduMzVXQWa4GQVSMZEUKD5x+TzGze4SZrpj5qIXreb5TArdTGtjtb+3r3HCe391UPZDUsmALoEsJIwmiXDCCW15DrlFy1GULgU8kThiewdHp0dt3DR0Xcko4/Qcx/RnUdl+rlYFOuZA4ImMxM9QPj09Kj+OUaTqqy7ugePhU3fQ4bg0PuJCxKkdvGgbjc6ErlskKUfwqxJQhXF9ftGhTtu3aQqeM6zNQWlYCEFtItXUGXgiuzeSx4ZS5pNZpd+wtx8raAfBFJCmz6uoWRCIlWTS+bh1Lr9OZYAbwQUgwrn9tB+7fylo67rB/e/Xr6O5vPTD/Xv31y/nN5ejubwc957uqSKQshDEWkFBd9fUFR0/ds8fvqXEklfU5HVvzmuXx6eH6fmg5Wg4ZUcZgXEU8I5LaoAhDPtWRI87TGKV3YXtmAxkZQRnKl79/vr20ASzz6S/mU6kbFepjhCSXtYVnBBliQGPClnEap7j7OJjCt72jvW8HK/T31X/sXZx9lZp8lRiOtE6+jin/Gi9IkgzwCff+81WDMiakIsxulPBDypil3QPKA5aGZgQiOsOeIW1FZH2T5p58/PfrT1/vPn+4//389urrJxpIocREf/3dxT7g5v7rRSolcv0bSkUF/zqMydSFg+HqCYO0Es0wf58tNPV1TrnpmhHJ10scp9NpycBngqnD60YyN4VNveVhJ5VGvmJUWyBWIz/dALzNpqcZqLIU6uZwimILSxhQvRh1t7iUunFhdvgt1v6T4FoiYU2wRMq1XIyoEqNAhDtB51jA8O4zGBYrQF6ct8DbleA8tBbZXRBOQtIAzfo51anntQLFyDpvqzlfCz6lOg1dDoARbT+ssnb/BXtM8L0z6P/5zeDt0cnPbw57sMeI3juDk9PB6eHpu6Of4b+bjJ5ZgQTf5fjeWg7rh7f/l4vV+HY0wB5by/j+JcUxBi3TfkIZDhIc0DiJiIo22OK2AizBe3UOhmYe+IwTIbUCyoHAlysbkB3AOQfPG/p9sx1wzaCCBszTgHCz1KbKOeATyqcoE2n2EWPKibSe8ww5kIlGCRIDESeUuWVXSBA6QmmHsc9whuV4vZaEq4mQsW2uICIzBBEEZnkKezCPaBDB3PowQUT4FCEWEs1rITVvEOZ663bq5fG4RiK5a080RFon6uz16/l8PphQibjAQSDi12Mmpq9dLKNvHAYig+j18eHRyevDo9dakuCR8mk/JmxOJPadnPqGp/GWIh2zQXGK5DpwGLz9+fBNcILvjo+PzH/CgJy+e/uGkPDN2zCcrNGOLVaF2hg2E2giUdhGMFV1ElfPnjUetEtOml69UpmiGfo9oBMgM0KZ8REHjTiUChGTnSBxpK24NkESh6c7gRGHpxtjUBE52o0sInL0HBTHp293heP49O1zkLz5+WRXSN78fPIcJKdHx7tCcnp0vAbJC+JO22yKM3iWfBMORf/RhKM97LOKyysFWmjCLNUmbs9Y6TdlWF3eM1b4pJGrZh9tG3453SamMY1x1FVIscD00/DTVWUI68tROcX+7GhNNY/zYtyXLlXj/ZpUsmXwYM8s6siI0jQYBGKvi4Hbfnogs6UrmlCeBaAZLnEsc3dC0inlNmbxPUWlG9BPJJnGWPH+dwP+i5DOdcsF7T078+nb//tWELsWSaOsJyljXQz5cGJJwcPttc3CeO+BcG080YVIpXFLISAKewbeohDvUlpIrFpdyuFbKtnAUP1m3Eu0zqmNMbkBo8p6sFxp6eothAQfSDJvGxnYkHmFcDXjWowEu8HtQh4PPBahzeIvdcaOjwKFttprCbABkvm7ERpdpoDyPNYdC061kJRPe04hs8qrh9triMnCxg/zobByk0iqhQZmi2GrQYCJqXKUDAGqQEw0cvh76grP8vopl+IkOqqivC8NSIx+xPN3c9pEAdWlarEemP0HQ23rTrhoTM8kRKma6Hc0nTyrbD75Od6MauNg4hqWOqqwK0za126P0zhxKwntlcu227Gd+djw82zLSmQnJ2+aMH1PUe4iUNdktC0vr3hhqR7LPfGx8KYeVKgZSX9/79ekBolnHL/92zej4vgUsDTEcLkoFBkOjCUkVuHzBYUL866dZbW6OGofFzpjCdiW5hmxXMepzlv1Cixd7/GJKl2d4TbIbnP8iV5is91w7b95GpWYQUgnE5TINSUaYYx6Xs+S2/TmXFhjrpr0wOU9UGI46s6pMNgjOo3QWqaMgTWqjknPdjNJMJ/AKh27R9Xh/CBktv/vFXI6lqCvM5kICXsTIQa+3SAQ8Z4Zkr3iF43W0AW7vWBD1ChjyjE0i1NAFbKFHx1gVGlg9BFdOVk6ZjQAlU4mtFpvaVvuR1onZ69fu4au3UDI6cEA7uXCZrkEkCSR4onGRPuaqPECFI0TtgBNHqsmwA2mLTM2I8rIGJlyKSIuNNglZ46MWXHcX1+qpXEKxCB9bDRNKohwJ/G8qkrcWUar7acNJDVDzDTjB1mrnJ+1295nc+v0Ar6nhDlXwbexNWQum1SrESSMZR02zaw9wsQts5FQ2r2c8tC7gbW5OAAYcruYS00JY9Wa2iqang06TnwpK2bPXfgbbJwwA2TdDss/IJyLqudVmg29gkxyS1nr3BiZmDfP0JY5XZ78RZG77QdRehAvPBmnx25mE6VrU9qZ42xoIqJcaj+xudKZmS5ismRW0D6Vjo8HKh0flUxIr2H+LaE6i+5dYy+WAqW9njMdXICWhDIz5xOUVISNEQaRjCzE51nhbXUdJxNfRKZF4hUkK97D++vLgx4QpgQ8cjHnRlJL8VZddWviemZscjNl1DbTkcJ0GdRtepV7hfhk2d4MjFWAfzKTbs35Kmu+HKZN7XqqUO4oU1PbPnlWK13xevDj6fTw3RbRD4WSEjZaWQa1bQ99oZRjk5U7UaXSZT154RQDkFRHQlLtq0zMNteYPx4sqgbEmuZcRY3PyJKI+NKNntlzLXfabjOQFWGIVEMgmDCGl4eQJglK49NVGAQRkSTQKNWKJNrp6Ydffnl38efLq18+HL77+fDd5dHxxcV5U0bXdngX4s2KDQwDM21WyPIikwRSG064pEpTPk2pijB0RPYvbw7Mknch4lhw/93FzUEPQkyQ2/oOwRv37Mt05/uHux58fn/l16MhD3rw+eG9XX2WNqcHFzd5m7uP58e2uh/OlUolKZ9nMH93Ztcsm1PlKh3/HYNdBJ2KZRxFqXqOYPYKP1y0d/fvL4wbIiSnpAfX7+8Ihw9GaFQFoiD6npH9wApaRURiOJgyMSYsHwaOTUE8wrQxQMY82uT0LsrXrs0C4HwHK8gCT+/97N+d3xwMnJxcCdmMyIUxF00Hntxfrux2ThcHzFakju2cN+Jni9zBWJ4BQNWDy5s7qPcZYN8QnFMWBkSGyqziPCzXkFfTussShj8VYr6vWoy4olNOdCq3PDPyyeWAYUJiyhZWyG6S7RezMg117WScKvQZ5LWLSQuA7MSskHBuSF58zDH5073DwhldaF2ZJpR1l1L4kKWCYJqaEbOLz8PtdUTSwuA1GZiGkXkxitL40EUr41QyA24UijlngnRSbn7tq3dg/+H2+sAFSWEhUuvhZYyAQCCShbN91B2Qa0U6ozJVNv01kKhStqk1boV6/ps/rGfgSlelvCEIs5w3JyImTJCW8qRWDGZD5QhvioNR/tjJqFH+mBXh/paT9+ejV8zlTVJiP2ge005U1+ywhpeFlM9Gk0eijeoFnczebBy82TAegFw7h+3h1nTTdXStCAJXFQuOaiaOh9vrAXzJjigWzgSB4Ixy7IGYTMx/nMPL7Ra0FbkrN+oKtT+IFQh71Eo4j8dqNFXgl53yAdwGSGNGgkeze1QDlcpxJ+mqu4fbX66XlL1YV8jStMDQipALPXIfN0WckNiZ846Ae3pw2YS/DZS/VmHTzcjasb2fU61RQkR4yAobV8fFJfIidyuBu86hOuiwLyQQLvgiFqk6aAXPiNTYZE/GQjAkfHPoQ+d7oSqkWLEEy4AeI/ICcpGHG20EKjsHva9lanN59pzKQeu8IhsfCWlfmlwcxUibTBUQpURAy8cMvqcoqTvqnvWpvlZwERNGu1oqHLU/xhLRcE/Ibspxyof4W5gmxEyUzpJCntymXZ4RRsPRRIq4AUBY3VC1cv89Ql5maPPb7pqZiUi5rUmw59G5MjPEHVSljdHarF5gV6hsRLCBydKkjJF1tj77TJJEVjwOmCNq0v2GZ1tBcAGxqkr0ioHfPK9hNkRU5RkPa5rnWUAjP8DSNGzNl4psYc8KjzfSZ3vLVCcyM4SyXLq7wWcN68AduOuC+dWTlsSefDRTJm86w4zJSlXa5Zne+9IsMkR6dotYVKExwl6mNzYa0rOb7I9ERf27j+fHp28bg+nCBn1G/li7cdA7m3v2IpLc4/ecGvZHbpVyPqnSEkm8k/Xvrkq65UoyaFgemxSPEaXyJGQXcqu7QYQXhp4qSCSdEVeqZCs/SJbdmaB3hvPAkn/CFvUDpyV3+iwj2vNvrAif+VatEmm6BCeThoqqxUJrdShGolK5PMEeBKkkwQL2bd8PzSw8Ojw8KN2JUxzuV0ZgGFKXYSKcsIWmgQKNQcQFE9OFIZELuN1VD1ETyo4belY7Kt1el2rpOCtj9LTRhhTkPoy9wzte2OQXnBy34vxB7lsTZ4t0pFApe96rm0sZLstHk8CTd3uZwAy23cYUbaTgJTVoB93Z1QslE7120u2RRPuU+F7PfbKGO/tAk+x/qWS1hP3eWGRNguMCmeB4+a19D/boceK/iwnLm1bpmWc5f/Nh+Uocnmb/9SQTIh8x9E2SiKooe7dK1j0sNvQkVEC4/97sxgtdUP6FKimVqsTdElZorIU0ja3lMx8kcj2yp7BQWlatYx8TtmL0swzDc4Y/P5YGEqdE2rAKyTd8LgvQAwIXv13B8NJWtBDuTrsRrUnwCPZSDqO3vcY95GazsHrw6IXqbFwHa5825yzRJc1GqaRdQLg1zpax3A+3wzoUb5DXJAxwhpLqTetB2292kFTTwN1l1jQ8Pq5mV2J/2V2SMFq1TmstAxPzHsT+prWITqMezFAu+ua/7Z210u+iq3f+qtLVww/FJercrKyFcwgrsY2McLpbGJb4NE6lVYpq2rYJiq7uZF8qJUPIg2i+C7UkJXsJaav/pGWqNIajgMqA4YiGnXjjyxXU0wdH35f1x3kkcNN5niYh0djRMD5YYjC8XBe774TZ7fUzbZqdlJ0lPi/tDS6FS5k2two06eWRAXuYyW/z4vC0vqvyi86Y/IOQ+iWZL9tPZSmnXyzRP2IGuYyw+Zjhj8gf/yECzK7WoAlU8erHBnDPO/id5amrdRorDoKsPci6vOc5p+x2ul7a9bEt2qXdIPF0N8ZhJmYjiLpom4W7JPUrcpSVS/lgzVHhtT1bqm52Cmzq2IBxxUG7OxCrFimDNLzfNZrh/UoghbCRy1A1KPDzDg0XbszzcVhfpTMnClQ6jql2eUbPkDVO9ECEODLWpRNHU4RoTZXZQdDCviJbuqpze3nuV9Xvv3yhtR/effnfSh2day3pOK14BaXL8oKOvI/SZXUXIo5Tbtz7pcu08sZGIae7wLABZ7fP8EX1jRCee2zejrKrUrc27tRG1vy5TTjKSt9joTQEfhfUvLJMRCcKEOa3tWTBeFraXpeC8CsFZcOZKuomRZ0ntNxpWWMdcvqNdeppd4G4h4eKWtTC6aXrDptW9edl7/JfhxATdxNhlbPtvw++NdrnbAqP6tfqv0RHP4o5xIQvloT9AW/uz0Ha+4rcNYhrRWR/QUKTOOlETjm1FwsrpMp1qqM0wmWBnnOSCzPGGPbGySJFIlQWBRwxETSVyz172tyhtj/94XID01RiCIK75cUmsGzpruHmblClau3wmcYdlZ34gnJfxmVBlQaOKs+t7Wy4swKj7rTqXMPcbM3tz7I0KlOTZStcLSMpn47sut/RElXOBljKGDpYCvKoi1q7dIVUkTEzRkFalWxW+JeMY4GizaE7RmF2LW9taFde+RGq0Y8w3UA1xPaSccd1hYsx6A6JqLgZTatoi3AMmC4LdUT1muCXANrVSJXAvQCXvUu7QwvVYJQIKySBJcZC13/Hoewyd6hLq91myv0lfI12Ky/eK/8oRgVnl2q2Aqk/3Lw91i41cAVWq5hdYO1aK1fgtXVemYKqBQ+yayo20NPcvetQWZfTOqf+3Dm9hNVlUY8u3c6/CbTS+XIfAKbJ7KTnbuQjPLRHaNu7EBCN0+qPf21TBe/pbdGVvRt/s9C5/6G2xmqlwiiIFcmRl6ryEnPhBvRUC7PjDAhji0yRsytNhpd37RB3tTS1S7cdk23SoUvofzLCDms3CNdshZ6/aSwDyqvIcz4Y9qo/0WeziSj98ejGu2UKmrijvWTJ3i87YGyrLRaIXmK8AhE/41K4NTFKS0pBTELMf2gjx0m1QjZpR7OrjYqlV9DH7Md73PcVncx/IWuJq11BkWFXhx6q06VwxiEWs+LlTpsP8Y/ZYeVIkZe2WR0ssL4kdRfL/2fvPzdPLbqcWRshbBHvVt5ALuS6cuTJvspOcv1ce87PKzyvpj3nUTk/n1e023tI9tz9mxE58onrvpLBXj2DIXTxhpJtEhif7//6f+Doy7/OG8C/zhv8s543+J8AAAD//2Eg0rs=" + return "eJzsfd1z2ziy7/v8FV2+D7G3JMV27OzEVXNveWxnRnUdJ+uPzNbe3FIgsiViDQIMAErRnjr/+yl8kOK3ZJvK2bO1fkkkQugfuhuNRncDHMIjrs5ARxKJplwj+wlAU82w+qVEhkThGUxRk58AQlSBpImmgp/B//4JAODe/gDsLxidIw8Q3lOGU/PtBxGmDEc/AcwoslCd2Z8MgZO4Rsv8hTgjKdMT2/oMZoQp9I/0KsEzmEuRJnnjGhjz995SgpkUMegIi1TWwOIcmPkrgisCpDykAdFCjmZUKj1RiDxvlEF6xNVSyLDwfQswxy2EkGgEwkPQNEZYRsjL3FMilQGCJQkSEyE1hqDoPNKUz0FHVBWQdYBmpA2zgdA7YEPuRXizn6gaXib4fDu8N2k8RQliZsGqCnVYEgViqlAuMIRA8DANPEirxSTQdEH1qgulQfRCJVglaBCuYRFlGCdRITe8m67gYmWG8dFCJVOGQDnc3Y//Csejw1Gpt6vvCQbmVwvCUlSlZwB/ApJqwUUsUjVUK6UxrreQms5IoGsPQiox0EKu6k9ETCgfGt7UnmFMKBuSMJS1RzPK6u1psjhpbk6TxdvmJzEJWh6kGr/Xvk2kCFDVeaPETC+JrGNKJat/p1AOSRCIlNdZtaQ8FEs1lDinSsvV8BHrXPs+PD18NwzQ8NtIHjvUrKA/L9O2S/toaucC2g6M7hlNFzybAk7n1tayc5YGhHOUE6WJfslMvTB8NFDOP7++uryFBfJQSIOSaFBpYAQ2SxlbQYjaaXhMGA2oSJVVJBASHm6vu7AmUixoiPJlHByHyI3EPAsNEWtgIizbwYxaF6JA8BkNTfNeMa27BUmsRSNK0TlfCzYDB6kyj60tKfxKBYSheppluREa7hIMDIxwADeC4wCuxXIAHzCkaTyA3+k8qv3scHh0WPvyPIypJIzqFdwZKLB/NHx7UGt2eTPOnp8O353WG/xx9SlrMI4ToRQ1xnMIFyg1ofygQzTOJ9iJqijHpcD7HW5VCokmQBUEIjYSMQ5Ll+aY5gr1bvF5ImvvqWGh7wJJkxq+0lfbQSPZelxaIQmH8ScwVh+Vgn0qJRrcmi7sGNxiRQU/6OSiXbh6Y2ILUr8+PhekcaFeYFm3wGcoAHe+0jNB2mV+5MWxa4Zy51W8VPoxkY+Uz0ea1fX0aS6cJDMzX66NywqfpNAiEAxURKSZy55Otz0FIvGsbsAiquv+yG+y7Ma7b8+N/Grf3mLYyQIdRBjWhl/cWUHDnmjdF9EipkGJbBsHO3lYs0au44LkrSvgAQMBJgLCAPmCSsFj5BqQh4mgxo2QwFEvhXwEXCDXowbcdkQ7gG37zRyCHY+hsvnoawiZV7izEdT1kNRtxvYq6CxXAyMqRrLChQdOv6WY2T3CzFDMfNTCjTzfKYHbKY3s9rf2dW44z+9uqh5IakmwFdA1h5EEUc4YwS2tMdcoOeqyhcDvJE4YnsHR6dHbdw0DF3JOOP0HMeMZ1XZf7cpA51xInJCpWJjeD49PSo/jlGk6qfO7oHj4vbrpcdQaHnAhY1WO3jQI42NhKJZICyt+E2LOEK6vLzq0Kdt2vUCnjOszUlpWAhAv4GrnDLwQXJvJY8MpS0mt0+7IW4qVtQPgk0hSZtXVLYhESrJq/Ll1LL1OZ4wZwXshwbj+tR24/1XW0lGH/dur3yZ3fxuA+ffqr5/Oby4nd387GDjfVUUiZSFMsYCE6qqvLzj63j15/JYaR1JZn9ORNT+zND48XN+PLUVLIeuUMZhWES+IpDYowpDPdeQ652mM0ruwA7OBjAyjTM+Xf3y8vbQBLPPpL+ZTaRiV3qcISc5rC88wMsSAxoSt4zROcfdxNIeve0d7Xw9a9PfV/9u7OPsiNfkiMZxonXyZUv4lXpEkGeF33Pv/rxqUMSEVZvajhO9TxmzfA6A8YGloJBDRBQ5M15ZF1jdpHsnv//f6w5e7j+/v/zi/vfrygQZSKDHTX/5wsQ+4uf9ykUqJXH9GqajgX8YxmbtwMFx9xyCtRDPM30cLTX1ZUm6GZljy5RKn6XxeMvAZY+rw+uHMTWFTb2nYSaWRt0i1A2I18tMPwNtsehpBlblQN4dzFC+whAHVq0l/i0tpGBdmh99h7T8IriUS1gRLcE05cr0rbEUdyIm14LwRUkdwbk0OaQSbci1XE6rEJBDhTljpSMD47iMYEi1IL8474O1Kyh5ah6AvCCdhE+esU1a1E16FUUysp9lO+VrwOdVp6BIWjGj7oc00/wfsMcH3zmD45zejt0cnP785HMAeI3rvDE5OR6eHp++Ofob/bLLQZrkUfJfyvbUUNot3+JeLdnw7ErDH1iHfv6Q4xaDDRs0ow1GCIxonEVHRFvvxToAleK/OwfSZR2njREitgHIg8OnKRo9HcM7B04bh0OxdXDOooAHzNCDc+AWpcruFGeVzlIk0m54p5URaN3+BHMhMowSJgYgTypyPICQIHaG0YhwyXGA5uaAl4WomZGybK4jIAkEEgVlLwwEsIxpEsLQOVxARPkeIhUTzs5CaXxDmRuvCCmV5XCOR3LUnGiKtE3X2+vVyuRzNqERc4SgQ8espE/PXLvAyNN4NkUH0+vjw6OT14dFrLUnwSPl8GBO2JBKHjk9DQ9O4dpGO2ag4RXIdOAze/nz4JjjBd8fHR+Y/YUBO3719Q0j45m0YzjZoxwuWsJoMmzto6qKw52Gq6tG2z54N7r7LpJpRvVKZopn+B0BnQBaEMuPQjhpxKBUiJjtB4rq27NoGSRye7gRGHJ5ujUFF5Gg3vIjI0VNQHJ++3RWO49O3T0Hy5ueTXSF58/PJU5CcHh3vCsnp0fEGJM8Ikr1kB5/Bs9034VD0H004umNUbVReKdBCE2Z7baL2hJV+W4LV5T0jhd81ctXso72EXt5vE9GYxjjpK/5ZIPph/OGqIsL6clSuB3hyaKmadHo27kuXV/J+TSrZOtKxZxZ1ZERpGowCsdeH4F4+PZDZOhtNKM+i5QzXONaJRiHpnHIbYPmWotIN6GeSzGOseP+7Af9JSOe65Yz2np359PV/fS2wXYukkdezlLE+RD6e2a7g4fbapoy890C4Np7oSqTSuKUQEIUDA29VCM4pLSRWrS7l8DWVbGR6/WrcS7TOqQ2IOYFRZT1YrrR0xSFCgo96mV8bHtj4fqXjanq4GLZ2wu2DHw88FqEtOVjrjJWPAoW2NG0NsAGS+bsRGl1ag/I8MB8LTrWQlM8HTiGzMrGH22uIycoGO3NRWL5JJNWqCLPFsKUrwMRcuZ5MB1SBmGnk8PfUVcnlxV4uH0t0VEV5XxJIjF7i+W/zvokCqkulbQMw+w+G2hbJcNGYS0qIUjXW72g6eVLZfPJzvBnV1pHPDSR1VCFXmLSv3R6nceJWsu+ty7bbsZ35QPbTbEsrspOTN02YvqUodxFVbDLalpZXvLBUPOae+MB90wgqvRlOf/vFr0kNHM8ofv0/X42K4/eApSGG60WhSHBkLCGxCp8vKFyY39pZVivio/ZxYTC2A9vSPCOW6jTVeatBgaQbPX6nSldnuM0I2IKERK+x2WG49l99H5WYQUhnM5TINSUaYYp6WU/p21zsUlhjrpr0wCVpUGI46c+pMNgjOo/QWqaMgDWqjsjADjNJMJ/AKp26R1Vxvhcy2/8PCgko26EvipkJCXszIUa+3SgQ8Z4RyV7xi0Zr6CLznrEhapQx5RiaxSmgCtnKSwcYVRoYfURX+5ZOGQ1ApbMZrRaH2pb7kdbJ2evXrqFrNxJyfjCCe7myKTkBJEmk+E5jon0B13QFisYJW4Emj1UT4IRpa6KNRBmZIlMun8WFBrvkLJExy47760u1Nk6BGKWPjaZJBRHuJJ5XVYk7S6jdftpAUjPETDN+kLXK6Vm77X02t06v4FtKmHMVfBtb8OZSX7WCRsJYNmDTzNojTNwyGwml3Y9THno3sDYXRwBjbhdzqSlhrFoAXEUzsEHHma+7xey5C3+DjRNmgKzbYekHhHNR9bxKs2FQ4EluKWuDmyITy+YZ2jGny5O/yHK3/SBKj+KV78bpsZvZROnalHbmOBNNRJSrQ0hsYndhpouYrYkVtE+l0+ORSqdHJRMyaJh/a6jOonvX2LOl0NPewJkOLkBLQpmZ8wlKKsLGCINIJhbi06zwS3UdZzNf8aZF4hUkqzTE++vLgwEQpgQ8crHkhlNr9lZddWviBkY2uZkyapvpSGG6jOo2vUq90vls3d4IxirAv5hJt+a8zZqvxbStXU8Vyh1lamrbJ0+q1RWvBz++nx6+e0H0Q6GkhE1aa7ZeOkJf1eXIZLVZVKl0XfxeOHIBJNWRkFT7khizzTXmjwerqgGxpjlXUeMzsiQivs5kYPZc65222wxkFSMi1RAIJozh5SGkSYLS+HQVAkFEJAk0StWSRDs9ff/rr+8u/nx59ev7w3c/H767PDq+uDhvyujaAe+CvVlW3BAw06aFlxcZJ5DacMIlVZryeUpVhKHrZP/y5sAseRcijgX3313cHAwgxAS5LUYRvHHPvk53/vJwN4CPv1z59WjMgwF8fPjFrj5rmzOAi5u8zd3v58f2KAKcK5VKUj58Yf7uzK5ZNqfKVTr9Owa7CDqV6g0KXPUUwewVfjhr7+5/uTBuiJCckgFc/3JHOLw3TKMqEAXWDwzvR5bRKiISw9GciSlhuRg4NgXxCNPGABnzaJPTu6i1uzYLgPMdLCMLNL33s393fnMwcnxy9W4LIlfGXDSdznJ/ubLbOV0UmC2fndo5b9jPVrmDsT6wgGoAlzd3UB8zwL7pcElZGBAZKrOK87Bc8F5N665LGP5UiPm+6jDiis450al84QGXDy4HDDMSU7ayTHaTbL+YlWkowifTVKHPIG9cTDoAZMd7hYRz0+XF7zkmfxR5XDhQDJ0r04yy/lIK77NUEMxTIzG7+DzcXkckLQivycA0SObZKEryoatOwqlkBtwkFEvOBOmlNv7aV+/A/sPt9YELksJKpNbDywgBgUAkK2f7qDvN14l0QWWqbPprJFGlbFtr3An1/LM/WWjgSldSvSUIs5w3JyJmTJCO8qRODGZD5TreFgej/LEXqVH+mFUMf86794e5W+byNimxHzSPaS+qa3ZY48tCymerySPRRvWCXmZvJgdvNowHIDfOYXsSN912Hd3IgsCV8ILrNWPHw+31CD5l5ykLB5hAcEY5DkDMZuY/zuHldgvaidyVG/WF2p8aC4Q9Fyacx2M1mirwy075tHADpCkjwaPZPaqRSuW0l3TV3cPtr9frnj1bW3hpWmBoWciFnriP2yJOSOzMeU/AfX9w2YS/C5S/A2LbzchG2d4vqdYoISI8ZIWNq6PiEnmRu0LB3T1RFTrsCwmEC76KRaoOOsEzIjU22ZOpEAwJ3x762PleqAopVizBMqCniLyAXOThRhuByg5t72uZ2lyePVRz0DmvyNbnV7qXJhdHMdwmcwVEKRHQ8pmIbylK6s7lZ2OqrxVcxITRvpYK19s/xxLRcKnJbspxyjcOdBBNiJkovSWFfHfbDnlBGA0nMyniBgBhdUPVSf2PCHmZoM1vuztxZiLltibBHp7nyswQd6qWNkZrs3qBXaGyEcEGImuTMkXW2/rsM0kSWfHsYo6oSfcbnr0IgguIVVViUAz85nkNsyGiKs94WNO8zAIa+WmbJrE134DyAntWeLyVPtsrsXrhmekoy6W764Y2kA7c6cA+iF9915LYY5pmyuRNF5gRaVWlXR5Avi/NItPJwG4Riyo0RdjL9MZGQwZ2k/07UdHw7vfz49O3jcF0YYM+E38G3zjovc09e2tK7vF7Sg37I7dKOZ9UaYkk3sn6d1ftuuP+NGhYHpsUjxGl8iRkH3yru0GEF0RPFSSSLogrVbKVHyTL7szQO8N5YMk/Yav66diSO32WdTrwv2gJn/lWnRxpurEn44aKqsVCG3UoRqJSuT5uHwSpJMEK9u3YD80sPDo8PChd4FMU9yvDMAypyzARTthK00CBxiDigon5ynSRM7jbVQ9RE8qOG0ZWO9fdXZdq+3FWxuhpow0p8H0ce4d3urLJLzg57sT5g9y3JsoW6UShUva8Vz83SFyWjyaB797tZQIjbLuNKdpIwUtq0A26t3siSiZ646TbI4n2KfG9gftkDXf2gSbZ/1LJagn7vanImgTHhW6C4/W39newR48T/11MWN602p95ltM3H9Y/icPT7L++y4TIRwx9kySiKsp+W+3WPSw29F2ogHD/vdmNF4ag/A+qXalUJe5Ks0JjLaRpbC2f+SCR64k9hYXSkuqUfUxYi/SzDMNTxJ8fSwOJcyJtWIXkGz6XBRgAgYvPVzC+tBUthLvTbkRrEjyCvUHE6O2gcQ+53SysHjx6pjob18Hap+0pS3RJs0kqaR8Qbo2zZSz3w+24DsUb5A0JA1ygpHrbetDuaygk1TRwF681icfH1exK7G/mSxJGq9Zpo2VgYjmA2F8LF9F5NIAFytXQ/Ld7sJb7fQz1zt+r2i5+KC5R52ZlLZxDaMU2Mczpb2FY49M4l1YpqmnbJii6upN9LpdMRx5E88WtJS7ZG1M7/SctU6UxnARUBgwnNOzFG1+voL5/cP37sv44jwRuO8/TJCQaexLjg+0MxpebYve9ELu9fqJNs5Oyt8Tnpb1upnCD1PZWgSaDPDJgDzP5bV4cntZ3VX7RmZJ/EFK/0fN5+6ks5fSr7fSfMYNcRth8zPBH5I//KQLMrtagCVTxnsoGcE87+J3lqat1Gi0HQTYeZF1fSp337Ha6ntt12Rbt0m6Q+H63xmEmZiOIOmubmbvu6jfkKCs3CMKGo8IbR7ZW3ewU2NyRAeOKg3YXNlYtUgZpfL9rNOP7ViCFsJHLUDUo8NMODReu9/NxWF+lsyQKVDqNqXZ5Rk+QNU70QIQ4MdalF0dThGhNldlB0MK+Ilu6qnN7fe5X1S/rfKa1H999+u9KHZ1rLek0rXgFpZv9gp68j9LNehcijlNu3Pu1y9R6vaSQ811g2IKy22f4ovpGCE89Nm+l7KrUrY07tZE1f24TjrLS91goDYHfBTWvLDPRiwKE+W0tWTCelrbXpSB8K6NsOFNF/aSo84SWOy1rrEPef2OdetpfIO7hoaIWtXB66W7GplX9adm7/FUWYuauTaxStuP3wbdG+5xN4Un9HQDP0dHfxRJiwlfrjv0Bb+7PQdr7itydjRtZZF93oUmc9MKnvLdnMyukyg2qpzTCZaE/5yQXZowx7I2TRYpEqCwKOGEiaCqXe/K0uUNt31PicgPzVGIIgrvlxSawbOmuoeaue6Vqo/hM457KTnxBuS/jsqBKgqPKU+s6G+6swKQ/rTrXsDRbc/sOmUZlarJshatlJOXziV33e1qiytkA2zOGDpaCPOqiNi5dIVVkyoxRkFYlmxX+OXIs9Ghz6I5QmN0hXBNt65UfoZr8CNMNVENsb0R3VFtcjFF/SETFzWhaRTuYY8D0WagjqncaPwfQriRVAvcMXPbi7x4tVINRIqyQBJYYC11/6UTZZe5Rl9rdZsr9JXyNdisv3iu/waOCs081a0HqDze/HGufGtiC1SpmH1j71soWvLbOK1NQteJBdk3FFnqau3c9Kut6Wue9P3VOr2H1WdSjS68S2AZa6Xy5DwDTZHEycDfyER7aI7TdQwiIxnn1TWUvqYL3/b1gKHs3/mahc/9WucZqpYIUREty5LmqvMZcuK491cLsOAPC2CpT5OxKk/HlXTfEXS1N3dztxmSb9OgS+vdbWLH2g3DDVujpm8YyoLyKPKeD4aD6PkGbTUTpj0c33i1T0MQd7SVL9n49AGNbbbFA9BzjFYj4CZfCbYhR2q4UxCTE/K0gOU6qFbJZN5pdbVRsfwV9zN405L6v6GT+Oq81rm4FRYZ9HXqoTpfCGYdYLIqXO20v4h+zw8qRIi9ts3pYYH1J6i6W/4/ef26eWnQ9s7ZC2MHeF3kDOZPrypEn+yo7yc1z7SnvgnhaTXtOo3J+Pq9ot/eQ7Ln7NyNy5BPXQyWDvXoGQ+jiDSUvSWB8vP/r/4CjL/8+bwD/Pm/wr37eYF0fGAgZYjhLGy9XqM/zzgr97czAracJ7y3R7U1CISxo7xapMKk9ob8h3X1le3O1V7njuX6ZYzXr3lwe0GqxiuDa8vFbZORzmOUEbPaXl7Tt0eTs6N3x6HB0PDp6s9eKshbR6Rsnt7dvfbbGL8uS+zthOvFvA742r/oG31QvXoY5Ts7d5SZ7jRpqK3ouquXK3RA3wCtec1CdQ+Pii24NWcdy0VkGmWNNJpLweZWjDmrLww1Yb+2LPsQMxp/UxnLM4oUwmfQPXx+9fdWIVlL12NfMv6Xq0c/np87yYF163KqJ01XtGp4tQOXACiRg/3B4clDXyAY012SK1Von6Gdy1FDZ05wj+MitrN0rtB94qlLCBnCXnw4YwIfs3ecD+Ixytf7cPiRcuHNL7nBOtYQMCneWEK2R1y4Kf8KYXqmcWr2EvYpLBaJ2Wy90HKl6Gndt70ba7951iNvMgbv6xc1FIC8S9LrQzoK6TRmur+d2d37PSGDP9cz8ix14/hOZ1t7F7v4KNv716Um7dbejS+OYyPap1T68DVdtbc0DJw4HY8NozKhPT4q88vlrtsq5NqqPt+vNpVWeNLyotMyQjgatbzQt8qPTsStjsQJulUxDWSc8VetkSeFGP/1XAAAA///mLJ1d" } diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/_meta/fields.yml b/x-pack/filebeat/module/threatintel/recordedfuture/_meta/fields.yml new file mode 100644 index 00000000000..3e4a82a2ec2 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/_meta/fields.yml @@ -0,0 +1,76 @@ +- name: recordedfuture + type: group + default_field: false + description: > + Fields for Recorded Future Threat Intel + fields: + - name: entity + type: group + description: > + Entity that represents a threat. + fields: + - name: id + type: keyword + description: > + Entity ID. + example: "ip:192.0.2.13" + - name: name + type: keyword + description: > + Entity name. Value for the entity. + example: "192.0.2.13" + - name: type + type: keyword + description: > + Entity type. + example: "IpAddress" + - name: intelCard + type: keyword + description: > + Link to the Recorded Future Intelligence Card for to this indicator. + - name: ip_range + type: ip_range + description: > + Range of IPs for this indicator. + example: '192.0.2.0/16' + - name: risk + type: group + description: > + Risk fields. + fields: + - name: criticality + type: byte + description: > + Risk criticality (0-4). + - name: criticalityLabel + type: keyword + description: > + Risk criticality label. One of None, Unusual, Suspicious, Malicious, Very Malicious. + - name: evidenceDetails + type: flattened + description: > + Risk's evidence details. + - name: score + type: short + description: > + Risk score (0-99). + - name: riskString + type: keyword + description: > + Number of Risk Rules observed as a factor of total number of rules. + example: "1/54" + - name: riskSummary + type: keyword + ignore_above: 1024 + description: > + Risk summary. + example: "1 of 54 Risk Rules currently observed." + multi_fields: + - name: text + type: text + norms: false + default_field: false + - name: rules + type: long + description: > + Number of rules observed. diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml b/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml new file mode 100644 index 00000000000..238d55d170c --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/config/config.yml @@ -0,0 +1,65 @@ +{{ if eq .input "httpjson" }} + +type: httpjson +config_version: "2" +interval: {{ .interval }} + +request.method: GET +{{ if .ssl }} +request.ssl: {{ .ssl | tojson }} +{{ end }} +{{ if .proxy_url }} +request.proxy_url: {{ .proxy_url }} +{{ end }} +request.url: "{{ .url }}&orderby=lastseen&direction=asc" +request.transforms: +{{ if .api_token }} +- set: + target: header.X-RFToken + value: {{ .api_token }} +- set: + target: url.params.lastSeen + value: '[[ .cursor.timestamp ]]' + default: '([[ formatDate (now (parseDuration "-{{ .first_interval }}")) "2006-01-02T15:04:05.000Z" ]],]' + {{ end }} +response.split: + target: body.data.results +cursor: + timestamp: + value: '([[ .first_event.timestamps.lastSeen ]],]' + +{{ else if eq .input "file" }} + +type: log +paths: +{{ range $i, $path := .paths }} + - {{$path}} +{{ end }} +exclude_files: [".gz$"] + +{{ end }} + +tags: {{.tags | tojson}} +publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} + +processors: + - decode_json_fields: + fields: [message] + target: json + - fingerprint: + fields: + - event.dataset + - json.entity.id + target_field: "@metadata._id" + encoding: base64 + - add_fields: + target: '' + fields: + ecs.version: 1.10.0 + - script: + lang: javascript + id: set_opt_type + source: > + function process(event) { + event.Put("@metadata.op_type", "index"); + } diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml b/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml new file mode 100644 index 00000000000..0a5e9937ed4 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/ingest/pipeline.yml @@ -0,0 +1,236 @@ +description: Pipeline for parsing Recorded Future threat intel. +processors: + # + # Safeguard against feeding the pipeline with documents other + # that the ones generated by Filebeat's httpjson input. + # + - fail: + if: 'ctx.json == null || !(ctx.json instanceof Map)' + message: 'missing json object in input document' + + # + # Set basic ECS fields. + # + - set: + field: event.ingested + value: '{{{ _ingest.timestamp }}}' + - set: + field: event.kind + value: enrichment + - set: + field: event.category + value: threat + - set: + field: event.type + value: indicator + + # + # Map itype field to STIX 2.0 Cyber Observable values (threatintel.indicator.type). + # + - script: + lang: painless + if: 'ctx.json.entity?.type != null' + description: > + Map entity.type field to STIX 2.0 Cyber Observable values (threatintel.indicator.type). + params: + IpAddress: ipv4-addr + InternetDomainName: domain-name + Hash: file + URL: url + source: > + String mapping = params[ctx.json.entity.type]; + if (mapping != null) { + ctx["threatintel_indicator_type"] = mapping; + } + on_failure: + - append: + field: error.message + value: 'Unable to determine indicator type from "{{{ json.entity.type }}}": {{{ _ingest.on_failure_message }}}' + + - rename: + field: threatintel_indicator_type + target_field: threatintel.indicator.type + ignore_missing: true + + # + # Detect ipv6 for ipv4-addr types. + # + - set: + field: threatintel.indicator.type + value: ipv6-addr + if: 'ctx.threatintel?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && ctx.json.entity.name.contains(":")' + + # + # Map first and last seen dates. + # + - date: + field: json.timestamps.firstSeen + target_field: threatintel.indicator.first_seen + formats: + - ISO8601 + if: 'ctx.json.timestamps?.firstSeen != null' + on_failure: + - append: + field: error.message + value: 'Error parsing firstSeen field value "{{{ json.timestamps.firstSeen }}}": {{{ _ingest.on_failure_message }}}' + - date: + field: json.timestamps.lastSeen + target_field: threatintel.indicator.last_seen + formats: + - ISO8601 + if: 'ctx.json.timestamps?.lastSeen != null' + on_failure: + - append: + field: error.message + value: 'Error parsing lastSeen field value "{{{ json.timestamps.lastSeen }}}": {{{ _ingest.on_failure_message }}}' + + + # + # Map location fields. + # + - rename: + field: json.location.location.city + target_field: threatintel.indicator.geo.city_name + ignore_missing: true + - rename: + field: json.location.location.continent + target_field: threatintel.indicator.geo.continent_name + ignore_missing: true + - rename: + field: json.location.location.country + target_field: threatintel.indicator.geo.country_name + ignore_missing: true + - grok: + field: json.location.asn + patterns: + - '^(?:[Aa][Ss])?%{NUMBER:threatintel.indicator.as.number:long}$' + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Cannot parse asn field `{{{ json.location.asn }}}`: {{{ _ingest.on_failure_message }}}' + + - rename: + field: json.location.organization + target_field: threatintel.indicator.as.organization.name + ignore_missing: true + + - set: + field: event.reference + value: '{{{ json.intelCard }}}' + ignore_empty_value: true + + - set: + field: json.ip_range + copy_from: json.entity.name + if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")' + - set: + field: json.ip_range + value: '{{{ json.entity.name }}}/32' + if: 'ctx.threatintel?.indicator?.type == "ipv4-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")' + - set: + field: json.ip_range + value: '{{{ json.entity.name }}}/128' + if: 'ctx.threatintel?.indicator?.type == "ipv6-addr" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")' + - set: + field: json.ip_range + copy_from: json.entity.name + if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && ctx.json.entity.name.contains("/")' + + - rename: + field: json.entity.name + target_field: threatintel.indicator.ip + if: 'ctx.json.entity?.type == "IpAddress" && ctx.json.entity.name != null && !ctx.json.entity.name.contains("/")' + + - rename: + field: json.entity.name + target_field: threatintel.indicator.domain + ignore_missing: true + if: 'ctx.threatintel?.indicator?.type == "domain-name"' + + - uri_parts: + field: json.entity.name + target_field: threatintel.indicator.url + keep_original: true + remove_if_successful: true + if: 'ctx.threatintel?.indicator?.type == "url"' + on_failure: + - append: + field: error.message + value: 'Cannot parse url field `{{{ json.entity.name }}}`: {{{ _ingest.on_failure_message }}}' + + # At this point fileHashes may exist if "fileHashes" field is requested. + - append: + field: json.fileHashes + value: '{{{ json.entity.name }}}' + allow_duplicates: false + if: 'ctx.threatintel?.indicator?.type == "file"' + + - remove: + field: json.entity.name + if: 'ctx.threatintel?.indicator?.type == "file"' + + - script: + lang: painless + description: > + Map file hashes. + if: 'ctx.json.fileHashes != null' + params: + '4': crc32 + '32': md5 + '40': sha1 + '64': sha256 + '128': sha512 + source: > + def hashes = new HashMap(); + for (def hash : ctx.json.fileHashes) { + def algo = params[String.valueOf(hash.length())]; + if (algo != null) { + hashes[algo] = hash; + } + } + ctx["_hashes"] = hashes; + on_failure: + - append: + field: error.message + value: 'Failed to map fileHashes field: {{ _ingest.on_failure_message }}' + + - rename: + field: _hashes + target_field: threatintel.indicator.file.hash + ignore_missing: true + + # + # Map risk.score to event.risk_score. + # + - convert: + field: json.risk.score + target_field: event.risk_score + ignore_missing: true + type: float + on_failure: + - append: + field: error.message + value: 'Risk score `{{{ json.risk.score }}}` cannot be converted to float: {{ _ingest.on_failure_message }}' + # + # Remove fields converted to an ECS field. + # + - remove: + field: + - json.timestamps + - json.location + - json.fileHashes + - message + ignore_missing: true + + # + # Save fields without an ECS mapping under `threatintel.recordedfuture`. + # + - rename: + field: json + target_field: threatintel.recordedfuture + +on_failure: + - append: + field: error.message + value: '{{ _ingest.on_failure_message }}' diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/manifest.yml b/x-pack/filebeat/module/threatintel/recordedfuture/manifest.yml new file mode 100644 index 00000000000..93df3884160 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/manifest.yml @@ -0,0 +1,19 @@ +module_version: 1.0 + +var: + - name: input + default: httpjson + - name: first_interval + default: 168h + - name: interval + default: 1m + - name: url + default: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + - name: ssl + - name: tags + default: [threatintel-recordedfuture, forwarded] + - name: proxy_url + - name: api_token +ingest_pipeline: + - ingest/pipeline.yml +input: config/config.yml diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log new file mode 100644 index 00000000000..54f047c3ab6 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "idn:16url-gy.example.net", "name": "16url-gy.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A16url-gy.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-07-25T20:29:32.750Z", "lastSeen": "2021-06-20T18:23:47.901Z"}} +{"entity": {"id": "idn:b999f.example.org", "name": "b999f.example.org", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ab999f.example.org", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2012-11-21T01:54:04.292Z", "lastSeen": "2021-06-20T18:23:47.812Z"}} +{"entity": {"id": "idn:c422.example.net", "name": "c422.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac422.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2018-02-21T13:53:46.470Z", "lastSeen": "2021-06-20T18:23:47.778Z"}} +{"entity": {"id": "idn:8rwcvgjsp.example.net", "name": "8rwcvgjsp.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A8rwcvgjsp.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-08-15T11:56:24.964Z", "lastSeen": "2021-06-20T18:23:47.747Z"}} +{"entity": {"id": "idn:c9px.example.net", "name": "c9px.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac9px.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-06-29T21:06:06.066Z", "lastSeen": "2021-06-20T18:23:47.460Z"}} +{"entity": {"id": "idn:ttj1i9z7.example.com", "name": "ttj1i9z7.example.com", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Attj1i9z7.example.com", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2018-09-20T03:26:08.564Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:7pgc.example.org", "name": "7pgc.example.org", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A7pgc.example.org", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2017-02-23T17:44:16.104Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:xm5u434.example.net", "name": "xm5u434.example.net", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Axm5u434.example.net", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2017-04-10T06:55:27.658Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:gpgju.example.com", "name": "gpgju.example.com", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Agpgju.example.com", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2018-07-27T15:22:39.390Z", "lastSeen": "2021-06-20T18:23:47.373Z"}} +{"entity": {"id": "idn:55g.example.com", "name": "55g.example.com", "type": "InternetDomainName"}, "intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A55g.example.com", "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/44", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-01-10T21:24:38.353Z", "lastSeen": "2021-06-20T18:23:45.025Z"}} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log-expected.json new file mode 100644 index 00000000000..12d7044c9a1 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/domain.ndjson.log-expected.json @@ -0,0 +1,312 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A16url-gy.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "16url-gy.example.net", + "threatintel.indicator.first_seen": "2016-07-25T20:29:32.750Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.901Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:16url-gy.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A16url-gy.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ab999f.example.org", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 482, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "b999f.example.org", + "threatintel.indicator.first_seen": "2012-11-21T01:54:04.292Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.812Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:b999f.example.org", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ab999f.example.org", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac422.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 955, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "c422.example.net", + "threatintel.indicator.first_seen": "2018-02-21T13:53:46.470Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.778Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:c422.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac422.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A8rwcvgjsp.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1425, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "8rwcvgjsp.example.net", + "threatintel.indicator.first_seen": "2016-08-15T11:56:24.964Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.747Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:8rwcvgjsp.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A8rwcvgjsp.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac9px.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1910, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "c9px.example.net", + "threatintel.indicator.first_seen": "2016-06-29T21:06:06.066Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.460Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:c9px.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Ac9px.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Attj1i9z7.example.com", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2380, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "ttj1i9z7.example.com", + "threatintel.indicator.first_seen": "2018-09-20T03:26:08.564Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:ttj1i9z7.example.com", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Attj1i9z7.example.com", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A7pgc.example.org", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2862, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "7pgc.example.org", + "threatintel.indicator.first_seen": "2017-02-23T17:44:16.104Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:7pgc.example.org", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A7pgc.example.org", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Axm5u434.example.net", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3332, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "xm5u434.example.net", + "threatintel.indicator.first_seen": "2017-04-10T06:55:27.658Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:xm5u434.example.net", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Axm5u434.example.net", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Agpgju.example.com", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3811, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "gpgju.example.com", + "threatintel.indicator.first_seen": "2018-07-27T15:22:39.390Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:47.373Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:gpgju.example.com", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3Agpgju.example.com", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A55g.example.com", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4284, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.domain": "55g.example.com", + "threatintel.indicator.first_seen": "2021-01-10T21:24:38.353Z", + "threatintel.indicator.last_seen": "2021-06-20T18:23:45.025Z", + "threatintel.indicator.type": "domain-name", + "threatintel.recordedfuture.entity.id": "idn:55g.example.com", + "threatintel.recordedfuture.entity.type": "InternetDomainName", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.com.local/live/sc/entity/idn%3A55g.example.com", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/44", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log new file mode 100644 index 00000000000..284429cc3e3 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "hash:dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "name": "dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "type": "Hash"}, "fileHashes": ["25328d1a481903f2d900479570842247", "d73c663e2ac0c7a14ca0e2681dd599b2e7a24f65", "dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Adec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://23l04ha7h.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.503Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://frj972mua.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:26.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.503Z", "lastSeen": "2021-06-20T18:40:18.503Z"}} +{"entity": {"id": "hash:4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "name": "4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "type": "Hash"}, "fileHashes": ["7b8d9afd032f0c253b7dd68aca6fb50b", "f9ece49c249aabab29fd9c2193d897b7d131ed17", "4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "8 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://poapoq2z.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.452Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://mezsa92p.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:27.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.452Z", "lastSeen": "2021-06-20T18:40:18.452Z"}} +{"entity": {"id": "hash:299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "name": "299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "type": "Hash"}, "fileHashes": ["7b65b50ed4554c86cb777e35e7750209", "e10942ba3fbb937c90c7cb3e39c06a13324981a8", "299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "9 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://kyvhpghg.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.343Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://fdxeziea.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:25.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.343Z", "lastSeen": "2021-06-20T18:40:18.343Z"}} +{"entity": {"id": "hash:e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "name": "e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "type": "Hash"}, "fileHashes": ["c6353df35499ca6934da2169b7bd1635", "3e208c649da0a9efbde7bbde6eece2142fdac3f9", "e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ae5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://k40z19-by.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.257Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://4e-6k-.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:29.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.258Z", "lastSeen": "2021-06-20T18:40:18.258Z"}} +{"entity": {"id": "hash:184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "name": "184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "type": "Hash"}, "fileHashes": ["3d568bd03766a8d47c8fabb7d392c32e", "3ea8b08bc9ed3009a4d6a0ab5851b8e3fc10ead2", "184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://ksmt6j.network.local/scan/results/file/184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:24.000Z"}], "riskString": "1/14", "riskSummary": "1 of 14 Risk Rules currently observed.", "rules": 1, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.131Z", "lastSeen": "2021-06-20T18:40:18.131Z"}} +{"entity": {"id": "hash:1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "name": "1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "type": "Hash"}, "fileHashes": ["a40e91f2d29616076114eea0f2a693af", "e38ccd47629c1b75385a83fbfbba0ea7f3b3a705", "1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "8 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://m-1z.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.093Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://llt6m.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:29.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.093Z", "lastSeen": "2021-06-20T18:40:18.093Z"}} +{"entity": {"id": "hash:bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "name": "bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "type": "Hash"}, "fileHashes": ["02062782c7eeaff185ea6966460f7c9a", "64355796dc38992ca5e434682ddbf63bdfabeb4e", "bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Abf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "4 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://46h0mn.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.070Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://j94d.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:28.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.070Z", "lastSeen": "2021-06-20T18:40:18.070Z"}} +{"entity": {"id": "hash:c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "name": "c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "type": "Hash"}, "fileHashes": ["bdd205ffc81c54e7cc1a9080cfa093e4", "a6b928fd6fee43495b96941ef80b25d074f6e0e2", "c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://5twber.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:18.010Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://b5qxg4.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:28.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:18.011Z", "lastSeen": "2021-06-20T18:40:18.011Z"}} +{"entity": {"id": "hash:c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "name": "c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "type": "Hash"}, "fileHashes": ["af45390e39574cdb037d684074e6a542", "f6a14c7424604cd51ba6a6d3f7594ec762f48645", "c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://wor2ca.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:17.964Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://l4tlgg.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:31.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:17.964Z", "lastSeen": "2021-06-20T18:40:17.964Z"}} +{"entity": {"id": "hash:0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "name": "0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "type": "Hash"}, "fileHashes": ["5b8bcd367f802cd104210bb47abb3ab1", "b40d1796bd6974860ce6be691152ad963300c711", "0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c"], "intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "risk": {"criticality": 3, "criticalityLabel": "Malicious", "evidenceDetails": [{"criticality": 2, "criticalityLabel": "Suspicious", "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://79073cr.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "mitigationString": "", "rule": "Linked to Malware", "timestamp": "2021-06-20T18:40:17.919Z"}, {"criticality": 3, "criticalityLabel": "Malicious", "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://c2ilj.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", "mitigationString": "", "rule": "Positive Malware Verdict", "timestamp": "2021-06-19T17:39:26.000Z"}], "riskString": "2/14", "riskSummary": "2 of 14 Risk Rules currently observed.", "rules": 2, "score": 65}, "timestamps": {"firstSeen": "2021-06-20T18:40:17.919Z", "lastSeen": "2021-06-20T18:40:17.919Z"}} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log-expected.json new file mode 100644 index 00000000000..32a800a1574 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/hash.ndjson.log-expected.json @@ -0,0 +1,494 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Adec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "25328d1a481903f2d900479570842247", + "threatintel.indicator.file.hash.sha1": "d73c663e2ac0c7a14ca0e2681dd599b2e7a24f65", + "threatintel.indicator.file.hash.sha256": "dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.503Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.503Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Adec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://frj972mua.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:26.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://23l04ha7h.network.local/scan/results/file/dec3a20fa1493c8e669b26d3f8b6084b34fda9906c978f9f12fb43f76504b5d6", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.503Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1478, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "7b8d9afd032f0c253b7dd68aca6fb50b", + "threatintel.indicator.file.hash.sha1": "f9ece49c249aabab29fd9c2193d897b7d131ed17", + "threatintel.indicator.file.hash.sha256": "4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.452Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.452Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://mezsa92p.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:27.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "8 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://poapoq2z.network.local/scan/results/file/4014355fdfee5fe9e01f3a84356d743c022cd75510f6c96ffe16fb332855d6f2", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.452Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2954, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "7b65b50ed4554c86cb777e35e7750209", + "threatintel.indicator.file.hash.sha1": "e10942ba3fbb937c90c7cb3e39c06a13324981a8", + "threatintel.indicator.file.hash.sha256": "299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.343Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.343Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://fdxeziea.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:25.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "9 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://kyvhpghg.network.local/scan/results/file/299e7a30217e2137854308e7be79227635f409b0e00897cfff11806ad8449cc5", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.343Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Ae5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4457, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "c6353df35499ca6934da2169b7bd1635", + "threatintel.indicator.file.hash.sha1": "3e208c649da0a9efbde7bbde6eece2142fdac3f9", + "threatintel.indicator.file.hash.sha256": "e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.258Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.258Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ae5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://4e-6k-.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:29.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://k40z19-by.network.local/scan/results/file/e5c73c63ba71659fbb9e0670cc203532aa61e3b8fa51f70ee5ce37b66784cd61", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.257Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 5932, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "3d568bd03766a8d47c8fabb7d392c32e", + "threatintel.indicator.file.hash.sha1": "3ea8b08bc9ed3009a4d6a0ab5851b8e3fc10ead2", + "threatintel.indicator.file.hash.sha256": "184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.131Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.131Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://ksmt6j.network.local/scan/results/file/184527a5436086cff0c06197330089f7964a9b6b8fc86327e6778363b7297ef1", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:24.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/14", + "threatintel.recordedfuture.risk.riskSummary": "1 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 7054, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "a40e91f2d29616076114eea0f2a693af", + "threatintel.indicator.file.hash.sha1": "e38ccd47629c1b75385a83fbfbba0ea7f3b3a705", + "threatintel.indicator.file.hash.sha256": "1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.093Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.093Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://llt6m.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:29.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "8 sightings on 1 source: PolySwarm. 1 related malware: Trojan. Most recent link (Jun 20, 2021): https://m-1z.network.local/scan/results/file/1136b8991c6f180a6c67eaff7c2a998d67dbcadc2d9cf5a3f816de03503817a8", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.093Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Abf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 8550, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "02062782c7eeaff185ea6966460f7c9a", + "threatintel.indicator.file.hash.sha1": "64355796dc38992ca5e434682ddbf63bdfabeb4e", + "threatintel.indicator.file.hash.sha256": "bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.070Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.070Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Abf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://j94d.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:28.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "4 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://46h0mn.network.local/scan/results/file/bf325093d87f746c297b2752c38a41a8f41b32aca01146b3632e24e90cdd14a1", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.070Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 10020, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "bdd205ffc81c54e7cc1a9080cfa093e4", + "threatintel.indicator.file.hash.sha1": "a6b928fd6fee43495b96941ef80b25d074f6e0e2", + "threatintel.indicator.file.hash.sha256": "c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "threatintel.indicator.first_seen": "2021-06-20T18:40:18.011Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:18.011Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://b5qxg4.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:28.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "3 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://5twber.network.local/scan/results/file/c06f58340d8e7b1f466942db18f67b5eb048c9adc45d843db370c836e125e3f9", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:18.010Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 11492, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "af45390e39574cdb037d684074e6a542", + "threatintel.indicator.file.hash.sha1": "f6a14c7424604cd51ba6a6d3f7594ec762f48645", + "threatintel.indicator.file.hash.sha256": "c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "threatintel.indicator.first_seen": "2021-06-20T18:40:17.964Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:17.964Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3Ac878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://l4tlgg.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:31.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://wor2ca.network.local/scan/results/file/c878bdb6c62ace8f001f979f7c7b2c6b38d135ac1c69bfa63785bf86721619fc", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:17.964Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/hash%3A0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "event.risk_score": 65.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 12964, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.file.hash.md5": "5b8bcd367f802cd104210bb47abb3ab1", + "threatintel.indicator.file.hash.sha1": "b40d1796bd6974860ce6be691152ad963300c711", + "threatintel.indicator.file.hash.sha256": "0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "threatintel.indicator.first_seen": "2021-06-20T18:40:17.919Z", + "threatintel.indicator.last_seen": "2021-06-20T18:40:17.919Z", + "threatintel.indicator.type": "file", + "threatintel.recordedfuture.entity.id": "hash:0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "threatintel.recordedfuture.entity.type": "Hash", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/hash%3A0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "threatintel.recordedfuture.risk.criticality": 3, + "threatintel.recordedfuture.risk.criticalityLabel": "Malicious", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 3, + "criticalityLabel": "Malicious", + "evidenceString": "1 sighting on 1 source: PolySwarm. Most recent link (Jun 19, 2021): https://c2ilj.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "mitigationString": "", + "rule": "Positive Malware Verdict", + "timestamp": "2021-06-19T17:39:26.000Z" + }, + { + "criticality": 2, + "criticalityLabel": "Suspicious", + "evidenceString": "6 sightings on 1 source: PolySwarm. Most recent link (Jun 20, 2021): https://79073cr.network.local/scan/results/file/0996575c7d2f07513d0dafe67ddde9805bbea35cf9d98edf8faf12c0e7f4334c", + "mitigationString": "", + "rule": "Linked to Malware", + "timestamp": "2021-06-20T18:40:17.919Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "2/14", + "threatintel.recordedfuture.risk.riskSummary": "2 of 14 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 2, + "threatintel.recordedfuture.risk.score": 65 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log new file mode 100644 index 00000000000..bb05454a584 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "ip:2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", "name": "2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", "location": {"asn": "AS31287", "cidr": {"id": "ip:151.237.36.0/23", "name": "151.237.36.0/23", "type": "IpAddress"}, "location": {"city": "Radnevo", "continent": "Europe", "country": "Bulgaria"}, "organization": "IPACCT CABLE Ltd"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-04-18T00:11:48.512Z", "lastSeen": "2021-06-19T19:40:32.897Z"}} +{"entity": {"id": "ip:2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", "name": "2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", "location": {"asn": "AS197207", "cidr": {"id": "ip:93.110.128.0/17", "name": "93.110.128.0/17", "type": "IpAddress"}, "location": {"city": null, "continent": "Asia", "country": "Iran"}, "organization": "Mobile Communication Company of Iran PLC"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T17:55:58.019Z", "lastSeen": "2021-06-19T19:40:32.839Z"}} +{"entity": {"id": "ip:203.0.113.55", "name": "203.0.113.55", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.55", "location": {"asn": null, "cidr": {"id": "ip:0.0.0.0/8", "name": "0.0.0.0/8", "type": "IpAddress"}, "location": {"city": null, "continent": null, "country": null}, "organization": null}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T19:40:30.596Z", "lastSeen": "2021-06-19T19:40:30.596Z"}} +{"entity": {"id": "ip:203.0.113.108", "name": "203.0.113.108", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.108", "location": {"asn": "AS17622", "cidr": {"id": "ip:58.248.128.0/19", "name": "58.248.128.0/19", "type": "IpAddress"}, "location": {"city": "Guangzhou", "continent": "Asia", "country": "China"}, "organization": "China Unicom Guangzhou network"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T19:40:20.534Z", "lastSeen": "2021-06-19T19:40:20.534Z"}} +{"entity": {"id": "ip:203.0.113.139", "name": "203.0.113.139", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.139", "location": {"asn": "AS7713", "cidr": {"id": "ip:125.162.0.0/16", "name": "125.162.0.0/16", "type": "IpAddress"}, "location": {"city": null, "continent": "Asia", "country": "Indonesia"}, "organization": "PT Telekomunikasi Indonesia"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2016-06-23T07:39:06.418Z", "lastSeen": "2021-06-19T19:40:03.882Z"}} +{"entity": {"id": "ip:2001:db8:bf58:c5c3:7a06:5267:82e0:621a", "name": "2001:db8:bf58:c5c3:7a06:5267:82e0:621a", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:bf58:c5c3:7a06:5267:82e0:621a", "location": {"asn": "AS17622", "cidr": {"id": "ip:58.249.64.0/19", "name": "58.249.64.0/19", "type": "IpAddress"}, "location": {"city": "Guangzhou", "continent": "Asia", "country": "China"}, "organization": "China Unicom Guangzhou network"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2021-06-19T19:40:02.557Z", "lastSeen": "2021-06-19T19:40:02.557Z"}} +{"entity": {"id": "ip:192.0.2.147", "name": "192.0.2.147", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.147", "location": {"asn": "AS4837", "cidr": {"id": "ip:61.53.0.0/17", "name": "61.53.0.0/17", "type": "IpAddress"}, "location": {"city": "Zhengzhou", "continent": "Asia", "country": "China"}, "organization": "CHINA UNICOM China169 Backbone"}, "risk": {"criticality": 0, "criticalityLabel": "None", "evidenceDetails": [], "riskString": "0/54", "riskSummary": "No Risk Rules are currently observed.", "rules": 0, "score": 0}, "timestamps": {"firstSeen": "2017-12-20T02:21:07.734Z", "lastSeen": "2021-06-19T19:39:43.160Z"}} +{"entity": {"id": "ip:203.0.113.198", "name": "203.0.113.198", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.198", "location": {"asn": "AS9829", "cidr": {"id": "ip:59.93.20.0/22", "name": "59.93.20.0/22", "type": "IpAddress"}, "location": {"city": "Palakkad", "continent": "Asia", "country": "India"}, "organization": "National Internet Backbone"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Dec 24, 2019): https://6900dkn8.network.local/pg6pd9jx/ip=203.0.113.198", "mitigationString": "", "rule": "Historical Multicategory Blocklist", "timestamp": "2019-12-24T09:53:13.546Z"}], "riskString": "1/54", "riskSummary": "1 of 54 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2019-12-24T09:54:02.935Z", "lastSeen": "2021-06-19T19:39:25.532Z"}} +{"entity": {"id": "ip:192.0.2.179", "name": "192.0.2.179", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.179", "location": {"asn": "AS9829", "cidr": {"id": "ip:59.99.200.0/21", "name": "59.99.200.0/21", "type": "IpAddress"}, "location": {"city": "Bangalore", "continent": "Asia", "country": "India"}, "organization": "National Internet Backbone"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Mar 3, 2020): https://f0go.network.local/c1c3m9rsl/ip=192.0.2.179", "mitigationString": "", "rule": "Historical Multicategory Blocklist", "timestamp": "2020-03-03T08:08:07.521Z"}, {"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between Apr 7, 2020, and Apr 8, 2020.", "mitigationString": "", "rule": "Historically Reported in Threat List", "timestamp": "2021-06-21T19:53:19.897Z"}, {"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "High Risk activity in CIDR Block.", "mitigationString": "", "rule": "Recorded Future Predictive Risk Model", "timestamp": "2021-06-21T19:53:19.906Z"}], "riskString": "3/54", "riskSummary": "3 of 54 Risk Rules currently observed.", "rules": 3, "score": 15}, "timestamps": {"firstSeen": "2020-03-03T08:10:28.489Z", "lastSeen": "2021-06-19T19:39:11.694Z"}} +{"entity": {"id": "ip:192.0.2.245", "name": "192.0.2.245", "type": "IpAddress"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.245", "location": {"asn": "AS45899", "cidr": {"id": "ip:113.170.96.0/20", "name": "113.170.96.0/20", "type": "IpAddress"}, "location": {"city": "Long Phu", "continent": "Asia", "country": "Vietnam"}, "organization": "VNPT Corp"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between May 25, 2021, and May 25, 2021.", "mitigationString": "", "rule": "Historically Reported in Threat List", "timestamp": "2021-06-19T19:50:20.162Z"}], "riskString": "1/54", "riskSummary": "1 of 54 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-19T19:38:57.372Z", "lastSeen": "2021-06-19T19:38:57.372Z"}} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log-expected.json new file mode 100644 index 00000000000..ed121c0a418 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/ip.ndjson.log-expected.json @@ -0,0 +1,414 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 31287, + "threatintel.indicator.as.organization.name": "IPACCT CABLE Ltd", + "threatintel.indicator.first_seen": "2021-04-18T00:11:48.512Z", + "threatintel.indicator.geo.city_name": "Radnevo", + "threatintel.indicator.geo.continent_name": "Europe", + "threatintel.indicator.geo.country_name": "Bulgaria", + "threatintel.indicator.last_seen": "2021-06-19T19:40:32.897Z", + "threatintel.indicator.type": "ipv6-addr", + "threatintel.recordedfuture.entity.id": "ip:2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.entity.name": "2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.ip_range": "2001:db8:cdb4:ff33:c406:fcdc:6961:c8af/21", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 763, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 197207, + "threatintel.indicator.as.organization.name": "Mobile Communication Company of Iran PLC", + "threatintel.indicator.first_seen": "2021-06-19T17:55:58.019Z", + "threatintel.indicator.geo.city_name": null, + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "Iran", + "threatintel.indicator.ip": "2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "threatintel.indicator.last_seen": "2021-06-19T19:40:32.839Z", + "threatintel.indicator.type": "ipv6-addr", + "threatintel.recordedfuture.entity.id": "ip:2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:f800:5c3f:c9f8:fbf8:d537:9071", + "threatintel.recordedfuture.ip_range": "2001:db8:f800:5c3f:c9f8:fbf8:d537:9071/128", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.55", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1531, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.organization.name": null, + "threatintel.indicator.first_seen": "2021-06-19T19:40:30.596Z", + "threatintel.indicator.geo.city_name": null, + "threatintel.indicator.geo.continent_name": null, + "threatintel.indicator.geo.country_name": null, + "threatintel.indicator.ip": "203.0.113.55", + "threatintel.indicator.last_seen": "2021-06-19T19:40:30.596Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.55", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.55", + "threatintel.recordedfuture.ip_range": "203.0.113.55/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.108", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2161, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 17622, + "threatintel.indicator.as.organization.name": "China Unicom Guangzhou network", + "threatintel.indicator.first_seen": "2021-06-19T19:40:20.534Z", + "threatintel.indicator.geo.city_name": "Guangzhou", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "China", + "threatintel.indicator.ip": "203.0.113.108", + "threatintel.indicator.last_seen": "2021-06-19T19:40:20.534Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.108", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.108", + "threatintel.recordedfuture.ip_range": "203.0.113.108/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.139", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2851, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 7713, + "threatintel.indicator.as.organization.name": "PT Telekomunikasi Indonesia", + "threatintel.indicator.first_seen": "2016-06-23T07:39:06.418Z", + "threatintel.indicator.geo.city_name": null, + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "Indonesia", + "threatintel.indicator.ip": "203.0.113.139", + "threatintel.indicator.last_seen": "2021-06-19T19:40:03.882Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.139", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.139", + "threatintel.recordedfuture.ip_range": "203.0.113.139/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3532, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 17622, + "threatintel.indicator.as.organization.name": "China Unicom Guangzhou network", + "threatintel.indicator.first_seen": "2021-06-19T19:40:02.557Z", + "threatintel.indicator.geo.city_name": "Guangzhou", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "China", + "threatintel.indicator.ip": "2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "threatintel.indicator.last_seen": "2021-06-19T19:40:02.557Z", + "threatintel.indicator.type": "ipv6-addr", + "threatintel.recordedfuture.entity.id": "ip:2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A2001:db8:bf58:c5c3:7a06:5267:82e0:621a", + "threatintel.recordedfuture.ip_range": "2001:db8:bf58:c5c3:7a06:5267:82e0:621a/128", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.147", + "event.risk_score": 0.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4295, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 4837, + "threatintel.indicator.as.organization.name": "CHINA UNICOM China169 Backbone", + "threatintel.indicator.first_seen": "2017-12-20T02:21:07.734Z", + "threatintel.indicator.geo.city_name": "Zhengzhou", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "China", + "threatintel.indicator.ip": "192.0.2.147", + "threatintel.indicator.last_seen": "2021-06-19T19:39:43.160Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:192.0.2.147", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.147", + "threatintel.recordedfuture.ip_range": "192.0.2.147/32", + "threatintel.recordedfuture.risk.criticality": 0, + "threatintel.recordedfuture.risk.criticalityLabel": "None", + "threatintel.recordedfuture.risk.evidenceDetails": [], + "threatintel.recordedfuture.risk.riskString": "0/54", + "threatintel.recordedfuture.risk.riskSummary": "No Risk Rules are currently observed.", + "threatintel.recordedfuture.risk.rules": 0, + "threatintel.recordedfuture.risk.score": 0 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.198", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4972, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 9829, + "threatintel.indicator.as.organization.name": "National Internet Backbone", + "threatintel.indicator.first_seen": "2019-12-24T09:54:02.935Z", + "threatintel.indicator.geo.city_name": "Palakkad", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "India", + "threatintel.indicator.ip": "203.0.113.198", + "threatintel.indicator.last_seen": "2021-06-19T19:39:25.532Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:203.0.113.198", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A203.0.113.198", + "threatintel.recordedfuture.ip_range": "203.0.113.198/32", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Dec 24, 2019): https://6900dkn8.network.local/pg6pd9jx/ip=203.0.113.198", + "mitigationString": "", + "rule": "Historical Multicategory Blocklist", + "timestamp": "2019-12-24T09:53:13.546Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/54", + "threatintel.recordedfuture.risk.riskSummary": "1 of 54 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.179", + "event.risk_score": 15.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 5970, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 9829, + "threatintel.indicator.as.organization.name": "National Internet Backbone", + "threatintel.indicator.first_seen": "2020-03-03T08:10:28.489Z", + "threatintel.indicator.geo.city_name": "Bangalore", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "India", + "threatintel.indicator.ip": "192.0.2.179", + "threatintel.indicator.last_seen": "2021-06-19T19:39:11.694Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:192.0.2.179", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.179", + "threatintel.recordedfuture.ip_range": "192.0.2.179/32", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "4 sightings on 1 source: AbuseIP Database. Most recent link (Mar 3, 2020): https://f0go.network.local/c1c3m9rsl/ip=192.0.2.179", + "mitigationString": "", + "rule": "Historical Multicategory Blocklist", + "timestamp": "2020-03-03T08:08:07.521Z" + }, + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "High Risk activity in CIDR Block.", + "mitigationString": "", + "rule": "Recorded Future Predictive Risk Model", + "timestamp": "2021-06-21T19:53:19.906Z" + }, + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between Apr 7, 2020, and Apr 8, 2020.", + "mitigationString": "", + "rule": "Historically Reported in Threat List", + "timestamp": "2021-06-21T19:53:19.897Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "3/54", + "threatintel.recordedfuture.risk.riskSummary": "3 of 54 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 3, + "threatintel.recordedfuture.risk.score": 15 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.245", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 7483, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.as.number": 45899, + "threatintel.indicator.as.organization.name": "VNPT Corp", + "threatintel.indicator.first_seen": "2021-06-19T19:38:57.372Z", + "threatintel.indicator.geo.city_name": "Long Phu", + "threatintel.indicator.geo.continent_name": "Asia", + "threatintel.indicator.geo.country_name": "Vietnam", + "threatintel.indicator.ip": "192.0.2.245", + "threatintel.indicator.last_seen": "2021-06-19T19:38:57.372Z", + "threatintel.indicator.type": "ipv4-addr", + "threatintel.recordedfuture.entity.id": "ip:192.0.2.245", + "threatintel.recordedfuture.entity.type": "IpAddress", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/ip%3A192.0.2.245", + "threatintel.recordedfuture.ip_range": "192.0.2.245/32", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "Previous sightings on 1 source: Recorded Future Fast Flux DNS IP List. Observed between May 25, 2021, and May 25, 2021.", + "mitigationString": "", + "rule": "Historically Reported in Threat List", + "timestamp": "2021-06-19T19:50:20.162Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/54", + "threatintel.recordedfuture.risk.riskSummary": "1 of 54 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log new file mode 100644 index 00000000000..2016b319f60 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log @@ -0,0 +1,10 @@ +{"entity": {"id": "url:https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", "name": "https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: Qakbot, qbot, zip. Most recent link (Apr 15, 2021): https://jc6.network.local/hnwdzm1k3?url=cdb14", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-04-15T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj"} +{"entity": {"id": "url:https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", "name": "https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: Bitdefender. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on Feb 14, 2021.", "mitigationString": "", "rule": "Historically Detected Phishing Techniques", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61"} +{"entity": {"id": "url:https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", "name": "https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: gafgyt, elf. Most recent link (May 15, 2021): https://bm1.network.local/82p3t?url=12b56", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-05-15T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc"} +{"entity": {"id": "url:https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", "name": "https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://ois8bq4.network.local/4lf?url=3f7c2", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0"} +{"entity": {"id": "url:http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", "name": "http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: @Sh1ttyKids. Most recent tweet: https://8g7zl.network.local/v5hcdo?url=efed5", "mitigationString": "", "rule": "Historically Reported as a Defanged URL", "timestamp": "2020-06-24T12:01:33.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii"} +{"entity": {"id": "url:http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", "name": "http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: . Most recent link (Feb 14, 2021): https://w6l3t5s.network.local/dr2rg5o?url=7b29d", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont"} +{"entity": {"id": "url:http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", "name": "http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Nov 16, 2020): https://0c39b.network.local/tcxah?url=0d1b9", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2020-11-16T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513"} +{"entity": {"id": "url:https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", "name": "https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (May 15, 2021): https://i1-yo.network.local/8-w8hhq2p?url=f7769", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-05-15T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r"} +{"entity": {"id": "url:http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", "name": "http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Feb 14, 2021): https://ysn.network.local/5p-09h7b?url=ff1c3", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2"} +{"entity": {"id": "url:http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", "name": "http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", "type": "URL"}, "risk": {"criticality": 1, "criticalityLabel": "Unusual", "evidenceDetails": [{"criticality": 1, "criticalityLabel": "Unusual", "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://j-8uc.network.local/2odpjhia?url=1a2ba", "mitigationString": "", "rule": "Historically Detected Malware Distribution", "timestamp": "2021-02-14T00:00:00.000Z"}], "riskString": "1/25", "riskSummary": "1 of 25 Risk Rules currently observed.", "rules": 1, "score": 5}, "timestamps": {"firstSeen": "2021-06-20T00:00:00.000Z", "lastSeen": "2021-06-20T23:59:59.000Z"}, "intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1"} diff --git a/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log-expected.json b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log-expected.json new file mode 100644 index 00000000000..b341365e428 --- /dev/null +++ b/x-pack/filebeat/module/threatintel/recordedfuture/test/url.ndjson.log-expected.json @@ -0,0 +1,442 @@ +[ + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 0, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "d6s.example.net", + "threatintel.indicator.url.original": "https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "threatintel.indicator.url.path": "/nzy/vvc68ke", + "threatintel.indicator.url.query": "p5uxwn=1bj", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://d6s.example.net/nzy/vvc68ke?p5uxwn=1bj", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: Qakbot, qbot, zip. Most recent link (Apr 15, 2021): https://jc6.network.local/hnwdzm1k3?url=cdb14", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-04-15T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 874, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "ga7v9u.example.org", + "threatintel.indicator.url.original": "https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "threatintel.indicator.url.path": "/bnqv8e2v8/qb49", + "threatintel.indicator.url.query": "7kq=iw61", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://ga7v9u.example.org/bnqv8e2v8/qb49?7kq=iw61", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: Bitdefender. Detected malicious behavior from an endpoint agent via global telemetry. Last observed on Feb 14, 2021.", + "mitigationString": "", + "rule": "Historically Detected Phishing Techniques", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 1760, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "cdmw.example.net", + "threatintel.indicator.url.original": "https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "threatintel.indicator.url.path": "/c20fwa/wwn", + "threatintel.indicator.url.query": "dlz53=z6ovc", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://cdmw.example.net/c20fwa/wwn?dlz53=z6ovc", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: gafgyt, elf. Most recent link (May 15, 2021): https://bm1.network.local/82p3t?url=12b56", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-05-15T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 2627, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "4mne.example.local", + "threatintel.indicator.url.original": "https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "threatintel.indicator.url.path": "/ns2rk8f/wngtk2xz", + "threatintel.indicator.url.query": "vceuk7wl6=3p0", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://4mne.example.local/ns2rk8f/wngtk2xz?vceuk7wl6=3p0", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://ois8bq4.network.local/4lf?url=3f7c2", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 3524, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "z198hloc8.example.com", + "threatintel.indicator.url.original": "http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "threatintel.indicator.url.path": "/f8ih39/f6kou", + "threatintel.indicator.url.query": "f6-u3=uwhii", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://z198hloc8.example.com/f8ih39/f6kou?f6-u3=uwhii", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: @Sh1ttyKids. Most recent tweet: https://8g7zl.network.local/v5hcdo?url=efed5", + "mitigationString": "", + "rule": "Historically Reported as a Defanged URL", + "timestamp": "2020-06-24T12:01:33.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 4377, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "y484j-fb6.example.local", + "threatintel.indicator.url.original": "http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "threatintel.indicator.url.path": "/b97s24xf/prz", + "threatintel.indicator.url.query": "sg-x1do=4myont", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://y484j-fb6.example.local/b97s24xf/prz?sg-x1do=4myont", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: . Most recent link (Feb 14, 2021): https://w6l3t5s.network.local/dr2rg5o?url=7b29d", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 5272, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "sp2xyqq82.example.local", + "threatintel.indicator.url.original": "http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "threatintel.indicator.url.path": "/zxvm093/kat1rcz", + "threatintel.indicator.url.query": "vaev0aeod=rc0513", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://sp2xyqq82.example.local/zxvm093/kat1rcz?vaev0aeod=rc0513", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Nov 16, 2020): https://0c39b.network.local/tcxah?url=0d1b9", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2020-11-16T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 6187, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "zh4o7xc.example.com", + "threatintel.indicator.url.original": "https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "threatintel.indicator.url.path": "/-yiq/vg2whtxif", + "threatintel.indicator.url.query": "cb0-knk=s6poib5r", + "threatintel.indicator.url.scheme": "https", + "threatintel.recordedfuture.entity.id": "url:https://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttps://zh4o7xc.example.com/-yiq/vg2whtxif?cb0-knk=s6poib5r", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (May 15, 2021): https://i1-yo.network.local/8-w8hhq2p?url=f7769", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-05-15T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 7094, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "fiivf4s.example.org", + "threatintel.indicator.url.original": "http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "threatintel.indicator.url.path": "/8u2qi/86vfcfq7m", + "threatintel.indicator.url.query": "pfb2ensc0=h7imk8io2", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://fiivf4s.example.org/8u2qi/86vfcfq7m?pfb2ensc0=h7imk8io2", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, mips. Most recent link (Feb 14, 2021): https://ysn.network.local/5p-09h7b?url=ff1c3", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + }, + { + "event.category": "threat", + "event.dataset": "threatintel.recordedfuture", + "event.kind": "enrichment", + "event.module": "threatintel", + "event.reference": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "event.risk_score": 5.0, + "event.type": "indicator", + "fileset.name": "recordedfuture", + "input.type": "log", + "log.offset": 8007, + "service.type": "threatintel", + "tags": [ + "forwarded", + "threatintel-recordedfuture" + ], + "threatintel.indicator.first_seen": "2021-06-20T00:00:00.000Z", + "threatintel.indicator.last_seen": "2021-06-20T23:59:59.000Z", + "threatintel.indicator.type": "url", + "threatintel.indicator.url.domain": "abav9v.example.org", + "threatintel.indicator.url.original": "http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "threatintel.indicator.url.path": "/gj93q/7fs7", + "threatintel.indicator.url.query": "kcq7=pjaj1", + "threatintel.indicator.url.scheme": "http", + "threatintel.recordedfuture.entity.id": "url:http://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "threatintel.recordedfuture.entity.type": "URL", + "threatintel.recordedfuture.intelCard": "https://app.recordedfuture.local/live/sc/entity/url%3Ahttp://abav9v.example.org/gj93q/7fs7?kcq7=pjaj1", + "threatintel.recordedfuture.risk.criticality": 1, + "threatintel.recordedfuture.risk.criticalityLabel": "Unusual", + "threatintel.recordedfuture.risk.evidenceDetails": [ + { + "criticality": 1, + "criticalityLabel": "Unusual", + "evidenceString": "1 sighting on 1 source: URLHaus. Malware: elf, Mozi. Most recent link (Feb 14, 2021): https://j-8uc.network.local/2odpjhia?url=1a2ba", + "mitigationString": "", + "rule": "Historically Detected Malware Distribution", + "timestamp": "2021-02-14T00:00:00.000Z" + } + ], + "threatintel.recordedfuture.risk.riskString": "1/25", + "threatintel.recordedfuture.risk.riskSummary": "1 of 25 Risk Rules currently observed.", + "threatintel.recordedfuture.risk.rules": 1, + "threatintel.recordedfuture.risk.score": 5 + } +] \ No newline at end of file diff --git a/x-pack/filebeat/modules.d/threatintel.yml.disabled b/x-pack/filebeat/modules.d/threatintel.yml.disabled index 255cd30aa40..e150fe8835a 100644 --- a/x-pack/filebeat/modules.d/threatintel.yml.disabled +++ b/x-pack/filebeat/modules.d/threatintel.yml.disabled @@ -140,3 +140,35 @@ # # var.ssl_certificate: path/to/server_ssl_cert.pem # var.ssl_key: path/to/ssl_key.pem + + recordedfuture: + enabled: true + + # Input used for ingesting threat intel data + var.input: httpjson + + # The interval to poll the API for updates + var.interval: 5m + + # How far back in time to start fetching intelligence when run for the + # first time. Value must be in hours. Default: 168h (1 week). + var.first_interval: 168h + + # The URL used for Threat Intel API calls. + # Must include the `limit` parameter and at least `entity` and `timestamps` fields. + # See the Connect API Explorer for a list of possible parameters. + # + # For `ip` entities: + var.url: "https://api.recordedfuture.com/v2/ip/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `domain` entities: + # var.url: "https://api.recordedfuture.com/v2/domain/search?limit=200&fields=entity,timestamps,risk,intelCard,location&metadata=false" + + # For `hash` entities: + # var.url: "https://api.recordedfuture.com/v2/hash/search?limit=200&fields=entity,fileHashes,timestamps,risk,intelCard,location&metadata=false" + + # For `url` entities: + # var.url: "https://api.recordedfuture.com/v2/url/search?limit=200&fields=entity,timestamps,risk&metadata=false" + + # Set your API Token. + var.api_token: "" diff --git a/x-pack/heartbeat/monitors/browser/config.go b/x-pack/heartbeat/monitors/browser/config.go index 0da4eb0a860..3be6af640ac 100644 --- a/x-pack/heartbeat/monitors/browser/config.go +++ b/x-pack/heartbeat/monitors/browser/config.go @@ -13,7 +13,8 @@ import ( func DefaultConfig() *Config { return &Config{ - Sandbox: false, + Sandbox: false, + Screenshots: "on", } } @@ -27,6 +28,7 @@ type Config struct { // Id is optional for lightweight checks but required for browsers Id string `config:"id"` Sandbox bool `config:"sandbox"` + Screenshots string `config:"screenshots"` SyntheticsArgs []string `config:"synthetics_args"` } diff --git a/x-pack/heartbeat/monitors/browser/suite.go b/x-pack/heartbeat/monitors/browser/suite.go index d265c0a4b3f..5e3d286d460 100644 --- a/x-pack/heartbeat/monitors/browser/suite.go +++ b/x-pack/heartbeat/monitors/browser/suite.go @@ -75,6 +75,9 @@ func (s *Suite) extraArgs() []string { if s.suiteCfg.Sandbox { extraArgs = append(extraArgs, "--sandbox") } + if s.suiteCfg.Screenshots != "" { + extraArgs = append(extraArgs, "--screenshots", s.suiteCfg.Screenshots) + } return extraArgs } diff --git a/x-pack/heartbeat/monitors/browser/suite_test.go b/x-pack/heartbeat/monitors/browser/suite_test.go index c583492d39d..54e708c3e07 100644 --- a/x-pack/heartbeat/monitors/browser/suite_test.go +++ b/x-pack/heartbeat/monitors/browser/suite_test.go @@ -127,11 +127,21 @@ func TestExtraArgs(t *testing.T) { &Config{}, nil, }, + { + "default", + DefaultConfig(), + []string{"--screenshots", "on"}, + }, { "sandbox", &Config{Sandbox: true}, []string{"--sandbox"}, }, + { + "screenshots", + &Config{Screenshots: "off"}, + []string{"--screenshots", "off"}, + }, { "capabilities", &Config{SyntheticsArgs: []string{"--capability", "trace", "ssblocks"}}, diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index e82fd9baef6..77ceb22c157 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -24,6 +24,14 @@ stages: unitTest: mage: "mage build unitTest" stage: mandatory + goIntegTest: + mage: "mage goIntegTest" + withModule: true + stage: mandatory + pythonIntegTest: + mage: "mage pythonIntegTest" + withModule: true + stage: mandatory cloud: cloud: "mage build test" withModule: true ## run the ITs only if the changeset affects a specific module. diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/data.json b/x-pack/metricbeat/module/gcp/billing/_meta/data.json index a9b52f82898..efbb1088bec 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/data.json +++ b/x-pack/metricbeat/module/gcp/billing/_meta/data.json @@ -1,7 +1,8 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "cloud.account.id": "elastic-bi", - "cloud.account.name": "elastic-bi", + "cloud.account.id": "01475F-5B1080-1137E7", + "cloud.project.id": "elastic-bi", + "cloud.project.name": "elastic-containerlib-prod", "cloud.provider": "gcp", "event": { "dataset": "gcp.billing", @@ -10,10 +11,12 @@ }, "gcp": { "billing": { + "billing_account_id": "01475F-5B1080-1137E7", "cost_type": "regular", - "invoice_month": "202010", - "project_id": "elastic-fin-bi", - "total": 77.897328 + "invoice_month": "202106", + "project_id": "containerlib-prod-12763", + "project_name": "elastic-containerlib-prod", + "total": 4717.170681 } }, "metricset": { diff --git a/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go b/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go index 9c006fb30cc..7da63d3dc64 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go +++ b/x-pack/metricbeat/module/gcp/billing/billing_integration_test.go @@ -10,10 +10,27 @@ package billing import ( "testing" + "github.com/stretchr/testify/assert" + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp/metrics" ) +func TestFetch(t *testing.T) { + config := metrics.GetConfigForTest(t, "billing") + config["period"] = "24h" + config["dataset_id"] = "master_gcp" + + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} + func TestData(t *testing.T) { config := metrics.GetConfigForTest(t, "billing") config["period"] = "24h" diff --git a/x-pack/metricbeat/tests/system/test_xpack_base.py b/x-pack/metricbeat/tests/system/test_xpack_base.py index ae19590dd24..043540666c2 100644 --- a/x-pack/metricbeat/tests/system/test_xpack_base.py +++ b/x-pack/metricbeat/tests/system/test_xpack_base.py @@ -1,9 +1,11 @@ import os +import unittest import xpack_metricbeat import test_base from beat import common_tests +@unittest.skip("https://github.com/elastic/beats/issues/26536") class Test(xpack_metricbeat.XPackTest, test_base.Test, common_tests.TestExportsMixin): pass diff --git a/x-pack/osquerybeat/beater/config_plugin_test.go b/x-pack/osquerybeat/beater/config_plugin_test.go index 456836a0a59..c8c52371a38 100644 --- a/x-pack/osquerybeat/beater/config_plugin_test.go +++ b/x-pack/osquerybeat/beater/config_plugin_test.go @@ -11,9 +11,9 @@ import ( "path/filepath" "testing" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/google/go-cmp/cmp" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/testutil" ) diff --git a/x-pack/osquerybeat/beater/logger_plugin.go b/x-pack/osquerybeat/beater/logger_plugin.go index 83c510207f5..3160c4d64f3 100644 --- a/x-pack/osquerybeat/beater/logger_plugin.go +++ b/x-pack/osquerybeat/beater/logger_plugin.go @@ -8,8 +8,9 @@ import ( "context" "encoding/json" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/kolide/osquery-go/plugin/logger" + + "github.com/elastic/beats/v7/libbeat/logp" ) type SnapshotResult struct { diff --git a/x-pack/osquerybeat/beater/logger_plugin_test.go b/x-pack/osquerybeat/beater/logger_plugin_test.go index a501d8aebab..d5dd625df67 100644 --- a/x-pack/osquerybeat/beater/logger_plugin_test.go +++ b/x-pack/osquerybeat/beater/logger_plugin_test.go @@ -9,10 +9,10 @@ import ( "encoding/json" "testing" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/google/go-cmp/cmp" "github.com/kolide/osquery-go/plugin/logger" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/x-pack/osquerybeat/internal/testutil" ) diff --git a/x-pack/osquerybeat/beater/osquerybeat.go b/x-pack/osquerybeat/beater/osquerybeat.go index fef59f975ec..24a24623c08 100644 --- a/x-pack/osquerybeat/beater/osquerybeat.go +++ b/x-pack/osquerybeat/beater/osquerybeat.go @@ -19,6 +19,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" @@ -430,7 +431,7 @@ func (bt *osquerybeat) publishEvents(index, actionID, responseID string, hits [] event.Fields["response_id"] = responseID } if index != "" { - event.Meta = common.MapStr{"index": index} + event.Meta = common.MapStr{events.FieldMetaRawIndex: index} } bt.client.Publish(event) diff --git a/x-pack/osquerybeat/internal/osqd/osqueryd.go b/x-pack/osquerybeat/internal/osqd/osqueryd.go index e3de3e00913..f2e5f184bda 100644 --- a/x-pack/osquerybeat/internal/osqd/osqueryd.go +++ b/x-pack/osquerybeat/internal/osqd/osqueryd.go @@ -18,8 +18,9 @@ import ( "time" "github.com/dolmen-go/contextio" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/logp" ) const ( diff --git a/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go b/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go index 22b4f2e820f..758dce74500 100644 --- a/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go +++ b/x-pack/osquerybeat/internal/osqd/osqueryd_windows.go @@ -13,8 +13,9 @@ import ( "github.com/gofrs/uuid" ) + const ( - extensionName = "osquery-extension.exe" + extensionName = "osquery-extension.exe" ) func CreateSocketPath() (string, func(), error) { diff --git a/x-pack/osquerybeat/internal/osqdcli/client.go b/x-pack/osquerybeat/internal/osqdcli/client.go index ae582226573..ceb0fee340f 100644 --- a/x-pack/osquerybeat/internal/osqdcli/client.go +++ b/x-pack/osquerybeat/internal/osqdcli/client.go @@ -14,8 +14,9 @@ import ( "golang.org/x/sync/semaphore" - "github.com/elastic/beats/v7/libbeat/logp" "github.com/kolide/osquery-go" + + "github.com/elastic/beats/v7/libbeat/logp" ) const (