Resolve pipeline issues

elastic · Sep 15, 2022 · 8c25b62 · 8c25b62
1 parent ed7bf0d
commit 8c25b62
Show file tree

Hide file tree

Showing 6 changed files with 144 additions and 14 deletions.
diff --git a/packages/barracuda_cloudgen_firewall/_dev/deploy/docker/config/filebeat.docker.yml b/packages/barracuda_cloudgen_firewall/_dev/deploy/docker/config/filebeat.docker.yml
@@ -22,10 +22,13 @@ filebeat.inputs:
     - /sample_logs/firewall.log
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering
-  fields:
-    type: ngfw-act
-    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
-    product: ngfw
+  processors:
+  - add_fields:
+      target: ''
+      fields:
+        type: ngfw-act
+        sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+        product: ngfw
 # filestream is an input for collecting log messages from files.
 - type: filestream
   # Change to true to enable this input configuration.
@@ -35,10 +38,13 @@ filebeat.inputs:
     - /sample_logs/threat.log
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering
-  fields:
-    type: ngfw-threat
-    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
-    product: ngfw
+  processors:
+  - add_fields:
+      target: ''
+      fields:
+        type: ngfw-threat
+        sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+        product: ngfw
 # filestream is an input for collecting log messages from files.
 - type: filestream
   # Change to true to enable this input configuration.
@@ -48,10 +54,13 @@ filebeat.inputs:
     - /sample_logs/web.log
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering
-  fields:
-    type: ngfw-wf
-    sn: 4f94abdf7a8c465fa2cd76f680ecafd1
-    product: ngfw
+  processors:
+  - add_fields:
+      target: ''
+      fields:
+        type: ngfw-wf
+        sn: 4f94abdf7a8c465fa2cd76f680ecafd1
+        product: ngfw
 
 # ============================== Filebeat modules ==============================
 filebeat.config.modules:

diff --git a/packages/barracuda_cloudgen_firewall/_dev/deploy/docker/docker-compose.yml b/packages/barracuda_cloudgen_firewall/_dev/deploy/docker/docker-compose.yml
@@ -7,4 +7,4 @@ services:
     volumes:
       - ./sample_logs:/sample_logs:ro
       - ./config:/config:ro
-    command: /bin/sh -c "sleep 10s && filebeat -c /config/filebeat.docker.yml -e -d '*'"
+    command: /bin/sh -c "while (filebeat -c /config/filebeat.docker.yml test output); [ $$? -ne 0 ]; do echo ''; done  && filebeat -c /config/filebeat.docker.yml -e -d '*'"
diff --git a/...ges/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml b/...ges/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/default.yml
@@ -29,6 +29,11 @@ processors:
   - json:
       field: event.original
       target_field: json
+  - remove:
+      field:
+        - source.address
+        - '@timestamp'
+      ignore_missing: true
   - pipeline:
       name: '{{ IngestPipeline "firewall" }}'
       if: ctx.lumberjack?.type == 'ngfw-act'

diff --git a/...ages/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/threat.yml b/...ages/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/threat.yml
@@ -10,6 +10,7 @@ processors:
     - set:
         field: _tmp.timestamp
         value: "{{json.date}} {{json.time}}"
+        if: ctx['@timestamp'] == null
     - date:
         field: _tmp.timestamp
         target_field: '@timestamp'

diff --git a/packages/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/web.yml b/packages/barracuda_cloudgen_firewall/data_stream/log/elasticsearch/ingest_pipeline/web.yml
@@ -5,7 +5,7 @@ processors:
         field: json.timestamp
         target_field: '@timestamp'
         formats:
-            - UNIX
+          - UNIX_MS
     - rename:
         field: json.source_ip
         target_field: source.address

diff --git a/packages/barracuda_cloudgen_firewall/data_stream/log/sample_event.json b/packages/barracuda_cloudgen_firewall/data_stream/log/sample_event.json
@@ -0,0 +1,115 @@
+{
+    "@timestamp": "2018-05-15T09:50:04.000Z",
+    "agent": {
+        "ephemeral_id": "2e425aa8-8270-4c3f-8f78-0a161f19edfa",
+        "id": "11051ba1-17de-4a11-9bfb-187ed7dd5199",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "8.5.0"
+    },
+    "barracuda_cloudgen_firewall": {
+        "log": {
+            "app_rule": "\u003cApp\u003e:\u003cpass-no-match\u003e"
+        }
+    },
+    "data_stream": {
+        "dataset": "barracuda_cloudgen_firewall.log",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "destination": {
+        "address": "89.160.20.114",
+        "as": {
+            "number": 29518,
+            "organization": {
+                "name": "Bredband2 AB"
+            }
+        },
+        "geo": {
+            "city_name": "Linköping",
+            "continent_name": "Europe",
+            "country_iso_code": "SE",
+            "country_name": "Sweden",
+            "location": {
+                "lat": 58.4167,
+                "lon": 15.6167
+            },
+            "region_iso_code": "SE-E",
+            "region_name": "Östergötland County"
+        },
+        "ip": "89.160.20.114",
+        "port": 443
+    },
+    "ecs": {
+        "version": "8.3.0"
+    },
+    "elastic_agent": {
+        "id": "11051ba1-17de-4a11-9bfb-187ed7dd5199",
+        "snapshot": true,
+        "version": "8.5.0"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "category": [
+            "network"
+        ],
+        "dataset": "barracuda_cloudgen_firewall.log",
+        "ingested": "2022-08-26T00:36:55Z",
+        "kind": "event"
+    },
+    "http": {
+        "request": {
+            "method": "GET"
+        },
+        "response": {
+            "body": {
+                "bytes": 0
+            },
+            "status_code": 200
+        }
+    },
+    "input": {
+        "type": "lumberjack"
+    },
+    "network": {
+        "type": "ipv4"
+    },
+    "observer": {
+        "product": "ngfw",
+        "serial_number": "4f94abdf7a8c465fa2cd76f680ecafd1",
+        "type": "firewall",
+        "vendor": "Barracuda"
+    },
+    "related": {
+        "ip": [
+            "192.168.42.105",
+            "89.160.20.114"
+        ]
+    },
+    "rule": {
+        "name": "LAN-2-INTERNET"
+    },
+    "source": {
+        "address": "192.168.42.105",
+        "ip": "192.168.42.105",
+        "port": 50159
+    },
+    "tags": [
+        "barracuda_cloudgen_firewall-log",
+        "forwarded"
+    ],
+    "url": {
+        "domain": "clientservices.googleapis.com",
+        "original": "https://clientservices.googleapis.com/chrome-variations/seed?osname=win\u0026channel=stable\u0026milestone=66",
+        "path": "/chrome-variations/seed",
+        "query": "osname=win\u0026channel=stable\u0026milestone=66",
+        "scheme": "https"
+    },
+    "user_agent": {
+        "device": {
+            "name": "Other"
+        },
+        "name": "Other",
+        "original": "mozilla/5.0 (windows nt 6.1) applewebkit/537.36 (khtml, like gecko) chrome/66.0.3359.139 safari/537.36"
+    }
+}