Add sample data, inital framework for system tests

packages/barracuda_cloudgen_firewall/_dev/build/docs/README.md

@@ -6,6 +6,6 @@ This integration ingests and parses logs from Barracuda Cloudgen Firewall device
 
 This is the Barracuda Cloudgen Firewall `log` dataset.
 
-{{event "log"}}
+
 
 {{fields "log"}}

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/config/filebeat.docker.yml

@@ -0,0 +1,230 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.reference.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+# For more available modules and options, please see the filebeat.reference.yml sample
+# configuration file.
+
+# ============================== Filebeat inputs ===============================
+
+filebeat.inputs:
+# filestream is an input for collecting log messages from files.
+- type: filestream
+  # Change to true to enable this input configuration.
+  enabled: true
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    - /sample_logs/firewall.log
+  # Optional additional fields. These fields can be freely picked
+  # to add additional information to the crawled log files for filtering
+  fields:
+    type: ngfw-act
+    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+    product: ngfw
+# filestream is an input for collecting log messages from files.
+- type: filestream
+  # Change to true to enable this input configuration.
+  enabled: true
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    - /sample_logs/threat.log
+  # Optional additional fields. These fields can be freely picked
+  # to add additional information to the crawled log files for filtering
+  fields:
+    type: ngfw-threat
+    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+    product: ngfw
+# filestream is an input for collecting log messages from files.
+- type: filestream
+  # Change to true to enable this input configuration.
+  enabled: true
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    - /sample_logs/web.log
+  # Optional additional fields. These fields can be freely picked
+  # to add additional information to the crawled log files for filtering
+  fields:
+    type: ngfw-wf
+    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+    product: ngfw
+
+# ============================== Filebeat modules ==============================
+filebeat.config.modules:
+  # Glob pattern for configuration loading
+  path: ${path.config}/modules.d/*.yml
+
+  # Set to true to enable config reloading
+  reload.enabled: false
+
+  # Period on which files under path should be checked for changes
+  #reload.period: 10s
+
+# ======================= Elasticsearch template setting =======================
+
+# setup.template.settings:
+#   index.number_of_shards: 1
+  #index.codec: best_compression
+  #_source.enabled: false
+
+
+# ================================== General ===================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+# ================================= Dashboards =================================
+# These settings control loading the sample dashboards to the Kibana index. Loading
+# the dashboards is disabled by default and can be enabled either by setting the
+# options here or by using the `setup` command.
+#setup.dashboards.enabled: false
+
+# The URL from where to download the dashboards archive. By default this URL
+# has a value which is computed based on the Beat name and version. For released
+# versions, this URL points to the dashboard archive on the artifacts.elastic.co
+# website.
+#setup.dashboards.url:
+
+# =================================== Kibana ===================================
+
+# Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
+# This requires a Kibana endpoint configuration.
+# setup.kibana:
+
+  # Kibana Host
+  # Scheme and port can be left out and will be set to the default (http and 5601)
+  # In case you specify and additional path, the scheme is required: http://localhost:5601/path
+  # IPv6 addresses should always be defined as: https://[2001:db8::1]:5601
+  #host: "localhost:5601"
+
+  # Kibana Space ID
+  # ID of the Kibana Space into which the dashboards should be loaded. By default,
+  # the Default Space will be used.
+  #space.id:
+
+# =============================== Elastic Cloud ================================
+
+# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/).
+
+# The cloud.id setting overwrites the `output.elasticsearch.hosts` and
+# `setup.kibana.host` options.
+# You can find the `cloud.id` in the Elastic Cloud web UI.
+#cloud.id:
+
+# The cloud.auth setting overwrites the `output.elasticsearch.username` and
+# `output.elasticsearch.password` settings. The format is `<user>:<pass>`.
+#cloud.auth:
+
+# ================================== Outputs ===================================
+
+# Configure what output to use when sending the data collected by the beat.
+
+# ---------------------------- Elasticsearch Output ----------------------------
+# output.elasticsearch:
+#   # Array of hosts to connect to.
+#   hosts: ["localhost:9200"]
+
+  # Protocol - either `http` (default) or `https`.
+  #protocol: "https"
+
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
+  #username: "elastic"
+  #password: "changeme"
+
+# ------------------------------ Logstash Output -------------------------------
+output.logstash:
+  # The Logstash hosts
+  hosts: ["elastic-agent:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+# ================================= Processors =================================
+# processors:
+#   - add_host_metadata:
+#       when.not.contains.tags: forwarded
+#   - add_cloud_metadata: ~
+#   - add_docker_metadata: ~
+#   - add_kubernetes_metadata: ~
+
+# ================================== Logging ===================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publisher", "service".
+#logging.selectors: ["*"]
+
+# ============================= X-Pack Monitoring ==============================
+# Filebeat can export internal metrics to a central Elasticsearch monitoring
+# cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
+# reporting is disabled by default.
+
+# Set to true to enable the monitoring reporter.
+#monitoring.enabled: false
+
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
+
+# Uncomment to send the metrics to Elasticsearch. Most settings from the
+# Elasticsearch output are accepted here as well.
+# Note that the settings should point to your Elasticsearch *monitoring* cluster.
+# Any setting that is not set is automatically inherited from the Elasticsearch
+# output configuration, so if you have the Elasticsearch output configured such
+# that it is pointing to your Elasticsearch monitoring cluster, you can simply
+# uncomment the following line.
+#monitoring.elasticsearch:
+
+# ============================== Instrumentation ===============================
+
+# Instrumentation support for the filebeat.
+#instrumentation:
+    # Set to true to enable instrumentation of filebeat.
+    #enabled: false
+
+    # Environment in which filebeat is running on (eg: staging, production, etc.)
+    #environment: ""
+
+    # APM Server hosts to report instrumentation results to.
+    #hosts:
+    #  - http://localhost:8200
+
+    # API Key for the APM Server(s).
+    # If api_key is set then secret_token will be ignored.
+    #api_key:
+
+    # Secret token for the APM Server(s).
+    #secret_token:
+
+
+# ================================= Migration ==================================
+
+# This allows to enable 6.7 migration aliases
+#migration.6_to_7.enabled: true

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/docker-compose.yml

@@ -1,8 +1,10 @@
 version: '2.3'
 services:
-  barracuda-cloudgen-logfile:
-    image: alpine
+  barracuda-cloudgen-lumberjack:
+    image: docker.elastic.co/beats/filebeat:8.3.3
+    environment:
+      - BEAT_STRICT_PERMS=false
     volumes:
       - ./sample_logs:/sample_logs:ro
-      - ${SERVICE_LOGS_DIR}:/var/log
-    command: /bin/sh -c "cp /sample_logs/* /var/log/"
+      - ./config:/config:ro
+    command: /bin/sh -c "sleep 10s && filebeat -c /config/filebeat.docker.yml -e -d '*'"

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/sample_logs/firewall.log

@@ -0,0 +1 @@
+{"version":1,"timestamp":1606230141,"action":"End","duration":8436,"src_iface":"eth0","src_ip":"10.17.35.171","src_port":40532,"src_mac":"00:0c:29:9a:0a:78","dst_iface":"eth0","dst_ip":"67.43.156.78","dst_port":443,"dst_mac":"00:0c:29:00:d6:00","fw_rule":"BOX-LAN-2-INTERNET","app_rule":"<App>:ALL-APPS","fw_info":2007,"src_ip_nat":"10.17.35.175","dst_ip_nat":"67.43.156.100","fwd_bytes":7450,"rev_bytes":561503,"fwd_packets":129,"rev_packets":439,"ip_proto":6,"protos":["HTTPS direct","HTTPS","All HTTP protocols"],"apps":["Web browsing"]}

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/sample_logs/threat.log

@@ -0,0 +1,3 @@
+{"app_target":"eicar.exe","component":"firewall","date":"2018 05 15","description":"Eicar-Test-Signature","dst_ip":"10.0.6.96","operation":"Block","port":"443","severity":"Warning","src_ip":"10.17.35.169","threat_severity":"3","time":"15:42:27","timestamp":"2018-05-15T15:42:27+00:00","timezone":"+00:00","trans_proto":"TCP","type":"Virus","user":"user42"}
+{"app_target":"boese.pdf","component":"firewall","date":"2018 05 15","description":"ad43f5fc1d679c8d766824abb41b2b28b364c3c8;.pdf","dst_ip":"89.160.20.129","operation":"Block","port":"80","severity":"Warning","src_ip":"10.17.35.169","threat_severity":"3","time":"15:42:32","timestamp":"2018-05-15T15:42:32+00:00","timezone":"+00:00","trans_proto":"TCP","type":"ATD","user":"user42"}
+{"component":"firewall","date":"2018 05 15","description":"ID: 1054837 WEB Remote File Inclusion /etc/passwd","dst_ip":"89.160.20.130","ips_category":"Web Attack","operation":"Block","port":"80","severity":"Warning","src_ip":"10.17.35.169","threat_severity":"3","time":"15:46:06","timestamp":"2018-05-15T15:46:06+00:00","timezone":"+00:00","trans_proto":"TCP","type":"IPS","user":"user45"}

packages/barracuda_cloudgen_firewall/_dev/deploy/docker/sample_logs/web.log

@@ -0,0 +1,2 @@
+{"timestamp":1526383397000,"traffic_type":0,"action":0,"source_ip":"192.168.42.124","source_port":"50646","destination_ip":"175.16.199.12","destination_port":"443","method":"GET","status_code":"200","user_agent":"wget/1.19.2 (linux-gnu)","content_type":"text/html; charset=UTF-8","name":"https://www.heise.de/","size":59558,"domain":"www.heise.de","category":["79"],"user":"192.168.42.124","user_type":0,"fw_rule":"LAN-2-INTERNET","app_rule":"<App>:<pass-no-match>"}
+{"timestamp":1526377804000,"traffic_type":0,"action":0,"source_ip":"192.168.42.105","source_port":"50159","destination_ip":"89.160.20.114","destination_port":"443","method":"GET","status_code":"200","user_agent":"mozilla/5.0 (windows nt 6.1) applewebkit/537.36 (khtml, like gecko) chrome/66.0.3359.139 safari/537.36","content_type":"","name":"https://clientservices.googleapis.com/chrome-variations/seed?osname=win&channel=stable&milestone=66","size":0,"domain":"clientservices.googleapis.com","category":[],"user":"192.168.42.105","user_type":0,"fw_rule":"LAN-2-INTERNET","app_rule":"<App>:<pass-no-match>"}

packages/barracuda_cloudgen_firewall/changelog.yml

@@ -3,4 +3,4 @@
   changes:
     - description: initial release
       type: enhancement # can be one of: enhancement, bugfix, breaking-change
-      link: https://github.com/elastic/package-storage/pull/1
+      link: https://github.com/elastic/package-storage/pull/3796

packages/barracuda_cloudgen_firewall/data_stream/log/_dev/test/pipeline/test-firewall.log-config.yml

@@ -0,0 +1,7 @@
+fields:
+  tags:
+    - preserve_original_event
+  lumberjack:
+    type: ngfw-act
+    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+    product: ngfw

packages/barracuda_cloudgen_firewall/data_stream/log/_dev/test/pipeline/test-firewall.log-expected.json

@@ -0,0 +1,95 @@
+{
+    "expected": [
+        {
+            "@timestamp": "2020-11-24T15:02:21.000Z",
+            "barracuda_cloudgen_firewall": {
+                "log": {
+                    "app_rule": "\u003cApp\u003e:ALL-APPS"
+                }
+            },
+            "destination": {
+                "address": "67.43.156.78",
+                "as": {
+                    "number": 35908
+                },
+                "bytes": 561503,
+                "geo": {
+                    "continent_name": "Asia",
+                    "country_iso_code": "BT",
+                    "country_name": "Bhutan",
+                    "location": {
+                        "lat": 27.5,
+                        "lon": 90.5
+                    }
+                },
+                "ip": "67.43.156.78",
+                "mac": "00-0C-29-00-D6-00",
+                "nat": {
+                    "ip": "67.43.156.100"
+                },
+                "packets": 439,
+                "port": 443
+            },
+            "ecs": {
+                "version": "8.3.0"
+            },
+            "event": {
+                "action": "End",
+                "category": [
+                    "network"
+                ],
+                "duration": -153934592,
+                "kind": "event",
+                "original": "{\"version\":1,\"timestamp\":1606230141,\"action\":\"End\",\"duration\":8436,\"src_iface\":\"eth0\",\"src_ip\":\"10.17.35.171\",\"src_port\":40532,\"src_mac\":\"00:0c:29:9a:0a:78\",\"dst_iface\":\"eth0\",\"dst_ip\":\"67.43.156.78\",\"dst_port\":443,\"dst_mac\":\"00:0c:29:00:d6:00\",\"fw_rule\":\"BOX-LAN-2-INTERNET\",\"app_rule\":\"\u003cApp\u003e:ALL-APPS\",\"fw_info\":2007,\"src_ip_nat\":\"10.17.35.175\",\"dst_ip_nat\":\"67.43.156.100\",\"fwd_bytes\":7450,\"rev_bytes\":561503,\"fwd_packets\":129,\"rev_packets\":439,\"ip_proto\":6,\"protos\":[\"HTTPS direct\",\"HTTPS\",\"All HTTP protocols\"],\"apps\":[\"Web browsing\"]}",
+                "type": [
+                    "end"
+                ]
+            },
+            "network": {
+                "community_id": "1:HGU1tX9W2VUF5ND2ey3X6Niv/AQ=",
+                "iana_number": "6",
+                "transport": "tcp",
+                "type": "ipv4"
+            },
+            "observer": {
+                "egress": {
+                    "interface": {
+                        "name": "eth0"
+                    }
+                },
+                "ingress": {
+                    "interface": {
+                        "name": "eth0"
+                    }
+                },
+                "product": "ngfw",
+                "serial_number": "4f94abdf7a8c465fa2cd76f680ecafd1",
+                "type": "firewall",
+                "vendor": "Barracuda"
+            },
+            "related": {
+                "ip": [
+                    "10.17.35.171",
+                    "67.43.156.78"
+                ]
+            },
+            "rule": {
+                "name": "BOX-LAN-2-INTERNET"
+            },
+            "source": {
+                "address": "10.17.35.171",
+                "bytes": 7450,
+                "ip": "10.17.35.171",
+                "mac": "00-0C-29-9A-0A-78",
+                "nat": {
+                    "ip": "10.17.35.175"
+                },
+                "packets": 129,
+                "port": 40532
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
+        }
+    ]
+}