From 8323e0c20c0bc2ac90e8974128e27aa4014da9f8 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Wed, 9 Jun 2021 14:43:27 +0200 Subject: [PATCH] [7.x](backport #26164) o365: Support non-array Parameters and ExtendedProperties fields (#26169) These fields are documented as being an array of Name-value pairs. However, in some cases they appear as a string field, leading to mapping errors. This patch will perform the expected name-value conversion by creating a new key, "_raw" with the original field value, when the fields are not arrays. (cherry picked from commit 07fa7f699487a2afc13bf0474c83325ee7c77e91) --- CHANGELOG.next.asciidoc | 1 + .../module/o365/audit/config/pipeline.js | 6 +- .../module/o365/audit/test/str-params.log | 2 + .../audit/test/str-params.log-expected.json | 131 ++++++++++++++++++ 4 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 x-pack/filebeat/module/o365/audit/test/str-params.log create mode 100644 x-pack/filebeat/module/o365/audit/test/str-params.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 2e30c19881a..ebe763c9a1f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -304,6 +304,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609] - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674] - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675] +- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164] *Heartbeat* diff --git a/x-pack/filebeat/module/o365/audit/config/pipeline.js b/x-pack/filebeat/module/o365/audit/config/pipeline.js index dd3257d50c6..48072acd8eb 100644 --- a/x-pack/filebeat/module/o365/audit/config/pipeline.js +++ b/x-pack/filebeat/module/o365/audit/config/pipeline.js @@ -108,7 +108,11 @@ function makeObjFromNameValuePairArray(options) { return function(evt) { var src = evt.Get(options.from); var dict = {}; - if (src == null || !(src instanceof Array)) return; + if (src == null) return; + if (!(src instanceof Array)) { + evt.Put(options.to, {"_raw": src} ); + return; + } for (var i=0; i < src.length; i++) { var name, value; if (src[i] == null diff --git a/x-pack/filebeat/module/o365/audit/test/str-params.log b/x-pack/filebeat/module/o365/audit/test/str-params.log new file mode 100644 index 00000000000..d5856330b9b --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/str-params.log @@ -0,0 +1,2 @@ +{"OriginatingServer": "HE1PR0102MB3228 (15.20.2707.017)", "ClientAppId": "", "OrganizationName": "testsiem.onmicrosoft.com", "ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}", "Parameters": "-StartReceivedDate \"4/25/2021 7:00:00 AM\" -EndReceivedDate \"5/27/2021 7:00:00 AM\" -StartExpiresDate \"5/26/2021 7:00:00 AM\" -EndExpiresDate \"6/26/2021 7:00:00 AM\" -PageSize \"100\" -Page \"1\" -MyItems \"True\" -QuarantineTypes (\"Bulk\",\"Spam\",\"Phish\")", "Workload": "Exchange", "OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", "CreationTime": "2020-02-07T20:49:49", "AppId": "", "UserId": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", "UserType": 3, "Version": 1, "ResultStatus": "True", "ExternalAccess": true, "UserKey": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", "Operation": "Set-Mailbox", "Id": "1c7412a6-858d-49ff-3f93-08d7ac0f45bf", "RecordType": 1} +{"CreationTime":"2021-02-05T09:06:07","Id":"550ed0e2-27da-4cbc-9fb8-46add4018800","Operation":"UserLoggedIn","OrganizationId":"48622b8f-44d3-420c-b4a2-510c8165767e","RecordType":15,"ResultStatus":"Success","UserKey":"21119711-1517-43d4-8138-b537dafad016","UserType":0,"Version":1,"Workload":"AzureActiveDirectory","ClientIP":"79.159.11.115","ObjectId":"Unknown","UserId":"root@testsiem4.onmicrosoft.com","AzureActiveDirectoryEventType":1,"ExtendedProperties": "-Name \"foo\" -Description \"\" -HoldNames () -PublicFolderLocation () -ExchangeLocationExclusion () -IncludeUserAppContent \"True\" -SharePointLocationExclusion () -Force \"True\" -Language \"\" -SharePointLocation () -ExchangeLocation (\"All\") -ContentMatchQuery \"(c:c)(senderauthor=abc@foo.com)\"","ModifiedProperties":[],"Actor":[{"ID":"21119711-1517-43d4-8138-b537dafad016","Type":0},{"ID":"root@testsiem4.onmicrosoft.com","Type":5}],"ActorContextId":"48622b8f-44d3-420c-b4a2-510c8165767e","ActorIpAddress":"79.159.11.115","InterSystemsId":"df4c6d6c-4551-4f2d-8766-03700dfccb47","IntraSystemId":"550ed0e2-27da-4cbc-9fb8-46add4018800","SupportTicketId":"","Target":[{"ID":"Unknown","Type":0}],"TargetContextId":"48622b8f-44d3-420c-b4a2-510c8165767e","ApplicationId":"89bee1f7-5e6e-4d8a-9f3d-ecd601259da7","ErrorNumber":"0"} diff --git a/x-pack/filebeat/module/o365/audit/test/str-params.log-expected.json b/x-pack/filebeat/module/o365/audit/test/str-params.log-expected.json new file mode 100644 index 00000000000..764d9283acf --- /dev/null +++ b/x-pack/filebeat/module/o365/audit/test/str-params.log-expected.json @@ -0,0 +1,131 @@ +[ + { + "@timestamp": "2020-02-07T20:49:49.000Z", + "event.action": "Set-Mailbox", + "event.category": "web", + "event.code": "ExchangeAdmin", + "event.dataset": "o365.audit", + "event.id": "1c7412a6-858d-49ff-3f93-08d7ac0f45bf", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.provider": "Exchange", + "event.type": "info", + "fileset.name": "audit", + "host.id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "host.name": "testsiem.onmicrosoft.com", + "input.type": "log", + "log.offset": 0, + "o365.audit.AppId": "", + "o365.audit.ClientAppId": "", + "o365.audit.CreationTime": "2020-02-07T20:49:49", + "o365.audit.ExternalAccess": true, + "o365.audit.Id": "1c7412a6-858d-49ff-3f93-08d7ac0f45bf", + "o365.audit.ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}", + "o365.audit.Operation": "Set-Mailbox", + "o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "o365.audit.OrganizationName": "testsiem.onmicrosoft.com", + "o365.audit.OriginatingServer": "HE1PR0102MB3228 (15.20.2707.017)", + "o365.audit.Parameters._raw": "-StartReceivedDate \"4/25/2021 7:00:00 AM\" -EndReceivedDate \"5/27/2021 7:00:00 AM\" -StartExpiresDate \"5/26/2021 7:00:00 AM\" -EndExpiresDate \"6/26/2021 7:00:00 AM\" -PageSize \"100\" -Page \"1\" -MyItems \"True\" -QuarantineTypes (\"Bulk\",\"Spam\",\"Phish\")", + "o365.audit.RecordType": 1, + "o365.audit.ResultStatus": "True", + "o365.audit.UserId": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", + "o365.audit.UserKey": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", + "o365.audit.UserType": 3, + "o365.audit.Version": 1, + "o365.audit.Workload": "Exchange", + "organization.id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", + "organization.name": "testsiem.onmicrosoft.com", + "server.address": "HE1PR0102MB3228 (15.20.2707.017)", + "service.type": "o365", + "tags": [ + "forwarded" + ], + "user.id": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)" + }, + { + "@timestamp": "2021-02-05T09:06:07.000Z", + "client.address": "79.159.11.115", + "client.ip": "79.159.11.115", + "event.action": "UserLoggedIn", + "event.category": "authentication", + "event.code": "AzureActiveDirectoryStsLogon", + "event.dataset": "o365.audit", + "event.id": "550ed0e2-27da-4cbc-9fb8-46add4018800", + "event.kind": "event", + "event.module": "o365", + "event.outcome": "success", + "event.provider": "AzureActiveDirectory", + "event.type": [ + "authentication_success", + "start" + ], + "fileset.name": "audit", + "host.id": "48622b8f-44d3-420c-b4a2-510c8165767e", + "host.name": "testsiem4.onmicrosoft.com", + "input.type": "log", + "log.offset": 1014, + "network.type": "ipv4", + "o365.audit.Actor": [ + { + "ID": "21119711-1517-43d4-8138-b537dafad016", + "Type": 0 + }, + { + "ID": "root@testsiem4.onmicrosoft.com", + "Type": 5 + } + ], + "o365.audit.ActorContextId": "48622b8f-44d3-420c-b4a2-510c8165767e", + "o365.audit.ActorIpAddress": "79.159.11.115", + "o365.audit.ApplicationId": "89bee1f7-5e6e-4d8a-9f3d-ecd601259da7", + "o365.audit.AzureActiveDirectoryEventType": 1, + "o365.audit.ClientIP": "79.159.11.115", + "o365.audit.CreationTime": "2021-02-05T09:06:07", + "o365.audit.ErrorNumber": "0", + "o365.audit.ExtendedProperties._raw": "-Name \"foo\" -Description \"\" -HoldNames () -PublicFolderLocation () -ExchangeLocationExclusion () -IncludeUserAppContent \"True\" -SharePointLocationExclusion () -Force \"True\" -Language \"\" -SharePointLocation () -ExchangeLocation (\"All\") -ContentMatchQuery \"(c:c)(senderauthor=abc@foo.com)\"", + "o365.audit.Id": "550ed0e2-27da-4cbc-9fb8-46add4018800", + "o365.audit.InterSystemsId": "df4c6d6c-4551-4f2d-8766-03700dfccb47", + "o365.audit.IntraSystemId": "550ed0e2-27da-4cbc-9fb8-46add4018800", + "o365.audit.ObjectId": "Unknown", + "o365.audit.Operation": "UserLoggedIn", + "o365.audit.OrganizationId": "48622b8f-44d3-420c-b4a2-510c8165767e", + "o365.audit.RecordType": 15, + "o365.audit.ResultStatus": "Success", + "o365.audit.SupportTicketId": "", + "o365.audit.Target": [ + { + "ID": "Unknown", + "Type": 0 + } + ], + "o365.audit.TargetContextId": "48622b8f-44d3-420c-b4a2-510c8165767e", + "o365.audit.UserId": "root@testsiem4.onmicrosoft.com", + "o365.audit.UserKey": "21119711-1517-43d4-8138-b537dafad016", + "o365.audit.UserType": 0, + "o365.audit.Version": 1, + "o365.audit.Workload": "AzureActiveDirectory", + "organization.id": "48622b8f-44d3-420c-b4a2-510c8165767e", + "related.ip": "79.159.11.115", + "related.user": "root", + "service.type": "o365", + "source.as.number": 3352, + "source.as.organization.name": "Telefonica De Espana", + "source.geo.city_name": "Barcelona", + "source.geo.continent_name": "Europe", + "source.geo.country_iso_code": "ES", + "source.geo.country_name": "Spain", + "source.geo.location.lat": 41.3891, + "source.geo.location.lon": 2.1611, + "source.geo.region_iso_code": "ES-B", + "source.geo.region_name": "Barcelona", + "source.ip": "79.159.11.115", + "tags": [ + "forwarded" + ], + "user.domain": "testsiem4.onmicrosoft.com", + "user.email": "root@testsiem4.onmicrosoft.com", + "user.id": "root@testsiem4.onmicrosoft.com", + "user.name": "root" + } +] \ No newline at end of file