Improve ECS categorization fields for azure module

- activitylogs
  + add azure.activitylogs.result_type
  + set default_field: false
  + populate event.outcome with allowed values
  + set event.action
  + populate event.category with allowed values
  + set event.kind
  + set event.type
  + add support tickets example
  + add geoip for source.ip
  + add AS info for source.ip
  + add user.name
  + add user.full_name
  + add user.domain
- auditlogs
  + set default_field: false
  + add azure.auditlogs.category
  + populate event.outcome with allowed values
  + set event.action
  + set event.kind
- signinlogs
  + set default_field: false
  + set event.action
  + populate event.category with allowed values
  + set event.type
  + populate event.outcome with allowed values
  + add azure.signinlogs.category
  + add azure.signinlogs.result_type
  + set user.name
  + set user.domain
  + set user.full_name
  + set user.id
  + add geoip for source.ip
  + add AS info for source.ip

filebeat/docs/fields.asciidoc

@@ -2335,6 +2335,16 @@ type: keyword
 Operation name
 
 
+type: keyword
+
+--
+
+*`azure.activitylogs.result_type`*::
++
+--
+Result type
+
+
 type: keyword
 
 --
@@ -2355,6 +2365,16 @@ type: keyword
 Category
 
 
+type: keyword
+
+--
+
+*`azure.activitylogs.event_category`*::
++
+--
+Event Category
+
+
 type: keyword
 
 --
@@ -2393,6 +2413,16 @@ Fields for Azure audit logs.
 
 
 
+*`azure.auditlogs.category`*::
++
+--
+The category of the operation.  Currently, Audit is the only supported value.
+
+
+type: keyword
+
+--
+
 *`azure.auditlogs.operation_name`*::
 +
 --
@@ -2788,6 +2818,16 @@ type: keyword
 Result description
 
 
+type: keyword
+
+--
+
+*`azure.signinlogs.result_type`*::
++
+--
+Result type
+
+
 type: keyword
 
 --
@@ -2798,6 +2838,16 @@ type: keyword
 Identity
 
 
+type: keyword
+
+--
+
+*`azure.signinlogs.category`*::
++
+--
+Category
+
+
 type: keyword
 
 --

x-pack/filebeat/module/azure/activitylogs/_meta/fields.yml

@@ -1,6 +1,7 @@
 - name: activitylogs
   type: group
   release: beta
+  default_field: false
   description: >
     Fields for Azure activity logs.
   fields:
@@ -86,6 +87,10 @@
       type: keyword
       description: >
         Operation name
+    - name: result_type
+      type: keyword
+      description: >
+        Result type
     - name: result_signature
       type: keyword
       description: >
@@ -94,6 +99,10 @@
       type: keyword
       description: >
         Category
+    - name: event_category
+      type: keyword
+      description: >
+        Event Category
     - name: properties
       type: group
       description: >

x-pack/filebeat/module/azure/activitylogs/ingest/pipeline.yml

@@ -52,23 +52,35 @@ processors:
     ignore_missing: true
 - script:
     lang: painless
-    source: if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.eventCategory
-      != null) {ctx.eventCategory = ctx.azure.activitylogs.properties.eventCategory}
-      if (ctx.azure.activitylogs.properties != null && ctx.azure.activitylogs.properties.policies
-      != null)  { ctx.eventCategory = 'Policy'}  if (ctx.eventCategory == null) {ctx.eventCategory='Administrative'}
+    source: >-
+      if (ctx?.azure?.activitylogs?.properties?.eventCategory != null) {
+        ctx.azure.activitylogs.event_category = ctx.azure.activitylogs.properties.eventCategory;
+      }
+      else if (ctx?.azure?.activitylogs?.properties?.policies != null)  {
+        ctx.azure.activitylogs.event_category = 'Policy';
+      }
+      else {
+        ctx.azure.activitylogs.event_category = 'Administrative';
+      }
     ignore_failure: true
-- rename:
-    field: eventCategory
-    target_field: event.category
-    ignore_missing: true
 - rename:
     field: azure.activitylogs.resultType
-    target_field: event.outcome
+    target_field: azure.activitylogs.result_type
     ignore_missing: true
+- convert:
+    field: azure.activitylogs.result_type
+    target_field: event.outcome
+    type: string
+    if: "ctx?.azure?.activitylogs?.resultType != null && ctx.azure.activitylogs.resultType instanceof String && (ctx.azure.activitylogs.resultType.toLowerCase() == 'success' || ctx.azure.activitylogs.resultType.toLowerCase() == 'failure')"
 - rename:
     field: azure.activitylogs.operationName
     target_field: azure.activitylogs.operation_name
     ignore_missing: true
+- convert:
+    field: azure.activitylogs.operation_name
+    target_field: event.action
+    type: string
+    ignore_missing: true
 - rename:
     field: azure.activitylogs.resultSignature
     target_field: azure.activitylogs.result_signature
@@ -119,21 +131,24 @@ processors:
     ignore_missing: true
 - script:
     lang: painless
-    source: if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname']
-      != null) {  ctx.azure.activitylogs.identity.claims_initiated_by_user.surname
-      = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'];}
+    source: >-
+      if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'] != null) {
+        ctx.azure.activitylogs.identity.claims_initiated_by_user.surname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'];
+      }
     ignore_failure: true
 - script:
     lang: painless
-    source: if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name']
-      != null) {  ctx.azure.activitylogs.identity.claims_initiated_by_user.name =
-      ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'];}
+    source: >-
+      if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'] != null) {
+        ctx.azure.activitylogs.identity.claims_initiated_by_user.name = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'];
+      }
     ignore_failure: true
 - script:
     lang: painless
-    source: if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname']
-      != null) {  ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname
-      = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'];}
+    source: >-
+      if (ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'] != null) {
+        ctx.azure.activitylogs.identity.claims_initiated_by_user.givenname = ctx.azure.activitylogs.identity.claims['http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'];
+      }
     ignore_failure: true
 - set:
     if: ctx.azure.activitylogs.identity!= null && ctx.azure.activitylogs.identity.claims_initiated_by_user
@@ -144,11 +159,69 @@ processors:
     ignore_failure: true
 - script:
     lang: painless
-    source: 'if (ctx.azure.activitylogs.identity.claims != null) { ctx.temp_claims
-      = new HashMap(); for (String key : ctx.azure.activitylogs.identity.claims.keySet())
-      {  ctx.temp_claims[key.replace(''.'', ''_'')] = ctx.azure.activitylogs.identity.claims.get(key)
-      ;}ctx.azure.activitylogs.identity.claims = ctx.temp_claims; ctx.remove(''temp_claims'');}'
+    source: >-
+      if (ctx.azure.activitylogs.identity.claims != null) {
+        ctx.temp_claims = new HashMap();
+        for (String key : ctx.azure.activitylogs.identity.claims.keySet()) {
+          ctx.temp_claims[key.replace('.', '_')] = ctx.azure.activitylogs.identity.claims.get(key);
+        }
+        ctx.azure.activitylogs.identity.claims = ctx.temp_claims; ctx.remove('temp_claims');
+      }
     ignore_failure: true
+- script:
+    lang: painless
+    ignore_failure: true
+    params:
+      "write":
+        type:
+          - change
+      "read":
+        type:
+          - access
+      "delete":
+        type:
+          - deletion
+      "action":
+        type:
+          - change
+    source: >-
+      if (ctx?.azure?.activitylogs?.category == null) {
+        return;
+      }
+      def hm = new HashMap(params.get(ctx.azure.activitylogs.category.toLowerCase()));
+      hm.forEach((k, v) -> ctx.event[k] = v);
+- geoip:
+    field: source.ip
+    target_field: source.geo
+- geoip:
+    database_file: GeoLite2-ASN.mmdb
+    field: source.ip
+    target_field: source.as
+    properties:
+    - asn
+    - organization_name
+    ignore_missing: true
+- rename:
+    field: source.as.asn
+    target_field: source.as.number
+    ignore_missing: true
+- rename:
+    field: source.as.organization_name
+    target_field: source.as.organization.name
+    ignore_missing: true
+- grok:
+    field: azure.activitylogs.identity.claims_initiated_by_user.name
+    patterns:
+      - '%{USERNAME:user.name}@%{HOSTNAME:user.domain}'
+    ignore_missing: true
+- convert:
+    field: azure.activitylogs.identity.claims_initiated_by_user.fullname
+    target_field: user.full_name
+    type: string
+    ignore_missing: true
+- set:
+    field: event.kind
+    value: event
 - pipeline:
     name: '{< IngestPipeline "azure-shared-pipeline" >}'
 on_failure:

x-pack/filebeat/module/azure/activitylogs/test/activitylogs.log-expected.json

@@ -2,6 +2,7 @@
     {
         "@timestamp": "2019-10-24T00:13:46.355Z",
         "azure.activitylogs.category": "Action",
+        "azure.activitylogs.event_category": "Administrative",
         "azure.activitylogs.identity.authorization.action": "Microsoft.EventHub/namespaces/authorizationRules/listKeys/action",
         "azure.activitylogs.identity.authorization.evidence.principal_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53",
         "azure.activitylogs.identity.authorization.evidence.principal_type": "ServicePrincipal",
@@ -26,6 +27,7 @@
         "azure.activitylogs.identity.claims.ver": "1.0",
         "azure.activitylogs.operation_name": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION",
         "azure.activitylogs.result_signature": "Started.",
+        "azure.activitylogs.result_type": "Start",
         "azure.correlation_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53",
         "azure.resource.authorization_rule": "ROOTMANAGESHAREDACCESSKEY",
         "azure.resource.group": "SA-HEMA",
@@ -34,11 +36,14 @@
         "azure.resource.provider": "MICROSOFT.EVENTHUB",
         "azure.subscription_id": "8a4de8b5-095c-47d0-a96f-a75130c61d53",
         "cloud.provider": "azure",
-        "event.category": "Administrative",
+        "event.action": "MICROSOFT.EVENTHUB/NAMESPACES/AUTHORIZATIONRULES/LISTKEYS/ACTION",
         "event.dataset": "azure.activitylogs",
-        "event.duration": 0.0,
+        "event.duration": 0,
+        "event.kind": "event",
         "event.module": "azure",
-        "event.outcome": "Start",
+        "event.type": [
+            "change"
+        ],
         "fileset.name": "activitylogs",
         "geo.continent_name": "Europe",
         "geo.country_iso_code": "GB",
@@ -48,6 +53,10 @@
         "log.level": "Information",
         "log.offset": 0,
         "service.type": "azure",
+        "source.geo.continent_name": "Europe",
+        "source.geo.country_iso_code": "GB",
+        "source.geo.location.lat": 51.4964,
+        "source.geo.location.lon": -0.1224,
         "source.ip": "51.251.141.41",
         "tags": [
             "forwarded"

x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log

@@ -0,0 +1 @@
+{"time":"2015-01-21T22:14:26.9792776Z","resourceId":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","operationName":"microsoft.support/supporttickets/write","category":"Write","resultType":"Success","resultSignature":"Succeeded.Created","durationMs":2826,"callerIpAddress":"111.111.111.11","correlationId":"c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8","identity":{"authorization":{"scope":"/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841","action":"microsoft.support/supporttickets/write","evidence":{"role":"Subscription Admin"}},"claims":{"aud":"https://management.core.windows.net/","iss":"https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/","iat":"1421876371","nbf":"1421876371","exp":"1421880271","ver":"1.0","http://schemas.microsoft.com/identity/claims/tenantid":"1e8d8218-c5e7-4578-9acc-9abbd5d23315 ","http://schemas.microsoft.com/claims/authnmethodsreferences":"pwd","http://schemas.microsoft.com/identity/claims/objectidentifier":"2468adf0-8211-44e3-95xq-85137af64708","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/upn":"admin@contoso.com","puid":"20030000801A118C","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/nameidentifier":"9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname":"John","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname":"Smith","name":"John Smith","groups":"cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c","http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name":" admin@contoso.com","appid":"c44b4083-3bq0-49c1-b47d-974e53cbdf3c","appidacr":"2","http://schemas.microsoft.com/identity/claims/scope":"user_impersonation","http://schemas.microsoft.com/claims/authnclassreference":"1"}},"level":"Information","location":"global","properties":{"statusCode":"Created","serviceRequestId":"50d5cddb-8ca0-47ad-9b80-6cde2207f97c"}}

x-pack/filebeat/module/azure/activitylogs/test/supporttickets_write.log-expected.json

@@ -0,0 +1,74 @@
+[
+    {
+        "@timestamp": "2015-01-21T22:14:26.979Z",
+        "azure.activitylogs.category": "Write",
+        "azure.activitylogs.event_category": "Administrative",
+        "azure.activitylogs.identity.authorization.action": "microsoft.support/supporttickets/write",
+        "azure.activitylogs.identity.authorization.evidence.role": "Subscription Admin",
+        "azure.activitylogs.identity.authorization.scope": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841",
+        "azure.activitylogs.identity.claims.appid": "c44b4083-3bq0-49c1-b47d-974e53cbdf3c",
+        "azure.activitylogs.identity.claims.appidacr": "2",
+        "azure.activitylogs.identity.claims.aud": "https://management.core.windows.net/",
+        "azure.activitylogs.identity.claims.exp": "1421880271",
+        "azure.activitylogs.identity.claims.groups": "cacfe77c-e058-4712-83qw-f9b08849fd60,7f71d11d-4c41-4b23-99d2-d32ce7aa621c,31522864-0578-4ea0-9gdc-e66cc564d18c",
+        "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnclassreference": "1",
+        "azure.activitylogs.identity.claims.http://schemas_microsoft_com/claims/authnmethodsreferences": "pwd",
+        "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/objectidentifier": "2468adf0-8211-44e3-95xq-85137af64708",
+        "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/scope": "user_impersonation",
+        "azure.activitylogs.identity.claims.http://schemas_microsoft_com/identity/claims/tenantid": "1e8d8218-c5e7-4578-9acc-9abbd5d23315 ",
+        "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/givenname": "John",
+        "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/name": " admin@contoso.com",
+        "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/nameidentifier": "9vckmEGF7zDKk1YzIY8k0t1_EAPaXoeHyPRn6f413zM",
+        "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/surname": "Smith",
+        "azure.activitylogs.identity.claims.http://schemas_xmlsoap_org/ws/2005/05/identity/claims/upn": "admin@contoso.com",
+        "azure.activitylogs.identity.claims.iat": "1421876371",
+        "azure.activitylogs.identity.claims.iss": "https://sts.windows.net/72f988bf-86f1-41af-91ab-2d7cd011db47/",
+        "azure.activitylogs.identity.claims.nbf": "1421876371",
+        "azure.activitylogs.identity.claims.puid": "20030000801A118C",
+        "azure.activitylogs.identity.claims.ver": "1.0",
+        "azure.activitylogs.identity.claims_initiated_by_user.fullname": "John Smith",
+        "azure.activitylogs.identity.claims_initiated_by_user.givenname": "John",
+        "azure.activitylogs.identity.claims_initiated_by_user.name": " admin@contoso.com",
+        "azure.activitylogs.identity.claims_initiated_by_user.schema": "http://schemas.xmlsoap.org/ws/2005/05/identity/claims",
+        "azure.activitylogs.identity.claims_initiated_by_user.surname": "Smith",
+        "azure.activitylogs.operation_name": "microsoft.support/supporttickets/write",
+        "azure.activitylogs.properties.service_request_id": "50d5cddb-8ca0-47ad-9b80-6cde2207f97c",
+        "azure.activitylogs.properties.status_code": "Created",
+        "azure.activitylogs.result_signature": "Succeeded.Created",
+        "azure.activitylogs.result_type": "Success",
+        "azure.correlation_id": "c776f9f4-36e5-4e0e-809b-c9b3c3fb62a8",
+        "azure.resource.id": "/subscriptions/s1/resourceGroups/MSSupportGroup/providers/microsoft.support/supporttickets/115012112305841",
+        "azure.resource.provider": "microsoft.support/supporttickets/115012112305841",
+        "cloud.provider": "azure",
+        "event.action": "microsoft.support/supporttickets/write",
+        "event.dataset": "azure.activitylogs",
+        "event.duration": -1468967296,
+        "event.kind": "event",
+        "event.module": "azure",
+        "event.type": [
+            "change"
+        ],
+        "fileset.name": "activitylogs",
+        "geo.continent_name": "Asia",
+        "geo.country_iso_code": "JP",
+        "geo.location.lat": 35.69,
+        "geo.location.lon": 139.69,
+        "input.type": "log",
+        "log.level": "Information",
+        "log.offset": 0,
+        "service.type": "azure",
+        "source.as.number": 2516,
+        "source.as.organization.name": "KDDI CORPORATION",
+        "source.geo.continent_name": "Asia",
+        "source.geo.country_iso_code": "JP",
+        "source.geo.location.lat": 35.69,
+        "source.geo.location.lon": 139.69,
+        "source.ip": "111.111.111.11",
+        "tags": [
+            "forwarded"
+        ],
+        "user.domain": "contoso.com",
+        "user.full_name": "John Smith",
+        "user.name": "admin"
+    }
+]