Skip to content

Commit

Permalink
[7.x](backport #26164) o365: Support non-array Parameters and Extende…
Browse files Browse the repository at this point in the history 
…dProperties fields (#26169)

These fields are documented as being an array of Name-value pairs.
However, in some cases they appear as a string field, leading to mapping
errors.

This patch will perform the expected name-value conversion by creating a
new key, "_raw" with the original field value, when the fields are not
arrays.

(cherry picked from commit 07fa7f6)
  • Loading branch information
@mergify
mergify[bot] authored Jun 9, 2021
1 parent 7847d38 commit 8323e0c
Show file tree
Hide file tree
Showing 4 changed files with 139 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Fix `checkpoint.action_reason` when its a string, not a Long. {issue}25575[25575] {pull}25609[25609]
- Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
- Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]

*Heartbeat*

Expand Down
6 changes: 5 additions & 1 deletion x-pack/filebeat/module/o365/audit/config/pipeline.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,11 @@ function makeObjFromNameValuePairArray(options) {
return function(evt) {
var src = evt.Get(options.from);
var dict = {};
if (src == null || !(src instanceof Array)) return;
if (src == null) return;
if (!(src instanceof Array)) {
evt.Put(options.to, {"_raw": src} );
return;
}
for (var i=0; i < src.length; i++) {
var name, value;
if (src[i] == null
Expand Down
2 changes: 2 additions & 0 deletions x-pack/filebeat/module/o365/audit/test/str-params.log
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
{"OriginatingServer": "HE1PR0102MB3228 (15.20.2707.017)", "ClientAppId": "", "OrganizationName": "testsiem.onmicrosoft.com", "ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}", "Parameters": "-StartReceivedDate \"4/25/2021 7:00:00 AM\" -EndReceivedDate \"5/27/2021 7:00:00 AM\" -StartExpiresDate \"5/26/2021 7:00:00 AM\" -EndExpiresDate \"6/26/2021 7:00:00 AM\" -PageSize \"100\" -Page \"1\" -MyItems \"True\" -QuarantineTypes (\"Bulk\",\"Spam\",\"Phish\")", "Workload": "Exchange", "OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd", "CreationTime": "2020-02-07T20:49:49", "AppId": "", "UserId": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", "UserType": 3, "Version": 1, "ResultStatus": "True", "ExternalAccess": true, "UserKey": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)", "Operation": "Set-Mailbox", "Id": "1c7412a6-858d-49ff-3f93-08d7ac0f45bf", "RecordType": 1}
{"CreationTime":"2021-02-05T09:06:07","Id":"550ed0e2-27da-4cbc-9fb8-46add4018800","Operation":"UserLoggedIn","OrganizationId":"48622b8f-44d3-420c-b4a2-510c8165767e","RecordType":15,"ResultStatus":"Success","UserKey":"21119711-1517-43d4-8138-b537dafad016","UserType":0,"Version":1,"Workload":"AzureActiveDirectory","ClientIP":"79.159.11.115","ObjectId":"Unknown","UserId":"root@testsiem4.onmicrosoft.com","AzureActiveDirectoryEventType":1,"ExtendedProperties": "-Name \"foo\" -Description \"\" -HoldNames () -PublicFolderLocation () -ExchangeLocationExclusion () -IncludeUserAppContent \"True\" -SharePointLocationExclusion () -Force \"True\" -Language \"\" -SharePointLocation () -ExchangeLocation (\"All\") -ContentMatchQuery \"(c:c)(senderauthor=abc@foo.com)\"","ModifiedProperties":[],"Actor":[{"ID":"21119711-1517-43d4-8138-b537dafad016","Type":0},{"ID":"root@testsiem4.onmicrosoft.com","Type":5}],"ActorContextId":"48622b8f-44d3-420c-b4a2-510c8165767e","ActorIpAddress":"79.159.11.115","InterSystemsId":"df4c6d6c-4551-4f2d-8766-03700dfccb47","IntraSystemId":"550ed0e2-27da-4cbc-9fb8-46add4018800","SupportTicketId":"","Target":[{"ID":"Unknown","Type":0}],"TargetContextId":"48622b8f-44d3-420c-b4a2-510c8165767e","ApplicationId":"89bee1f7-5e6e-4d8a-9f3d-ecd601259da7","ErrorNumber":"0"}
131 changes: 131 additions & 0 deletions x-pack/filebeat/module/o365/audit/test/str-params.log-expected.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,131 @@
[
{
"@timestamp": "2020-02-07T20:49:49.000Z",
"event.action": "Set-Mailbox",
"event.category": "web",
"event.code": "ExchangeAdmin",
"event.dataset": "o365.audit",
"event.id": "1c7412a6-858d-49ff-3f93-08d7ac0f45bf",
"event.kind": "event",
"event.module": "o365",
"event.outcome": "success",
"event.provider": "Exchange",
"event.type": "info",
"fileset.name": "audit",
"host.id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"host.name": "testsiem.onmicrosoft.com",
"input.type": "log",
"log.offset": 0,
"o365.audit.AppId": "",
"o365.audit.ClientAppId": "",
"o365.audit.CreationTime": "2020-02-07T20:49:49",
"o365.audit.ExternalAccess": true,
"o365.audit.Id": "1c7412a6-858d-49ff-3f93-08d7ac0f45bf",
"o365.audit.ObjectId": "EURPR01A002.prod.outlook.com/Microsoft Exchange Hosted Organizations/testsiem.onmicrosoft.com/DiscoverySearchMailbox{D919BA05-46A6-415f-80AD-7E09334BB852}",
"o365.audit.Operation": "Set-Mailbox",
"o365.audit.OrganizationId": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"o365.audit.OrganizationName": "testsiem.onmicrosoft.com",
"o365.audit.OriginatingServer": "HE1PR0102MB3228 (15.20.2707.017)",
"o365.audit.Parameters._raw": "-StartReceivedDate \"4/25/2021 7:00:00 AM\" -EndReceivedDate \"5/27/2021 7:00:00 AM\" -StartExpiresDate \"5/26/2021 7:00:00 AM\" -EndExpiresDate \"6/26/2021 7:00:00 AM\" -PageSize \"100\" -Page \"1\" -MyItems \"True\" -QuarantineTypes (\"Bulk\",\"Spam\",\"Phish\")",
"o365.audit.RecordType": 1,
"o365.audit.ResultStatus": "True",
"o365.audit.UserId": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)",
"o365.audit.UserKey": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)",
"o365.audit.UserType": 3,
"o365.audit.Version": 1,
"o365.audit.Workload": "Exchange",
"organization.id": "b86ab9d4-fcf1-4b11-8a06-7a8f91b47fbd",
"organization.name": "testsiem.onmicrosoft.com",
"server.address": "HE1PR0102MB3228 (15.20.2707.017)",
"service.type": "o365",
"tags": [
"forwarded"
],
"user.id": "NT AUTHORITY\\SYSTEM (Microsoft.Exchange.ServiceHost)"
},
{
"@timestamp": "2021-02-05T09:06:07.000Z",
"client.address": "79.159.11.115",
"client.ip": "79.159.11.115",
"event.action": "UserLoggedIn",
"event.category": "authentication",
"event.code": "AzureActiveDirectoryStsLogon",
"event.dataset": "o365.audit",
"event.id": "550ed0e2-27da-4cbc-9fb8-46add4018800",
"event.kind": "event",
"event.module": "o365",
"event.outcome": "success",
"event.provider": "AzureActiveDirectory",
"event.type": [
"authentication_success",
"start"
],
"fileset.name": "audit",
"host.id": "48622b8f-44d3-420c-b4a2-510c8165767e",
"host.name": "testsiem4.onmicrosoft.com",
"input.type": "log",
"log.offset": 1014,
"network.type": "ipv4",
"o365.audit.Actor": [
{
"ID": "21119711-1517-43d4-8138-b537dafad016",
"Type": 0
},
{
"ID": "root@testsiem4.onmicrosoft.com",
"Type": 5
}
],
"o365.audit.ActorContextId": "48622b8f-44d3-420c-b4a2-510c8165767e",
"o365.audit.ActorIpAddress": "79.159.11.115",
"o365.audit.ApplicationId": "89bee1f7-5e6e-4d8a-9f3d-ecd601259da7",
"o365.audit.AzureActiveDirectoryEventType": 1,
"o365.audit.ClientIP": "79.159.11.115",
"o365.audit.CreationTime": "2021-02-05T09:06:07",
"o365.audit.ErrorNumber": "0",
"o365.audit.ExtendedProperties._raw": "-Name \"foo\" -Description \"\" -HoldNames () -PublicFolderLocation () -ExchangeLocationExclusion () -IncludeUserAppContent \"True\" -SharePointLocationExclusion () -Force \"True\" -Language \"\" -SharePointLocation () -ExchangeLocation (\"All\") -ContentMatchQuery \"(c:c)(senderauthor=abc@foo.com)\"",
"o365.audit.Id": "550ed0e2-27da-4cbc-9fb8-46add4018800",
"o365.audit.InterSystemsId": "df4c6d6c-4551-4f2d-8766-03700dfccb47",
"o365.audit.IntraSystemId": "550ed0e2-27da-4cbc-9fb8-46add4018800",
"o365.audit.ObjectId": "Unknown",
"o365.audit.Operation": "UserLoggedIn",
"o365.audit.OrganizationId": "48622b8f-44d3-420c-b4a2-510c8165767e",
"o365.audit.RecordType": 15,
"o365.audit.ResultStatus": "Success",
"o365.audit.SupportTicketId": "",
"o365.audit.Target": [
{
"ID": "Unknown",
"Type": 0
}
],
"o365.audit.TargetContextId": "48622b8f-44d3-420c-b4a2-510c8165767e",
"o365.audit.UserId": "root@testsiem4.onmicrosoft.com",
"o365.audit.UserKey": "21119711-1517-43d4-8138-b537dafad016",
"o365.audit.UserType": 0,
"o365.audit.Version": 1,
"o365.audit.Workload": "AzureActiveDirectory",
"organization.id": "48622b8f-44d3-420c-b4a2-510c8165767e",
"related.ip": "79.159.11.115",
"related.user": "root",
"service.type": "o365",
"source.as.number": 3352,
"source.as.organization.name": "Telefonica De Espana",
"source.geo.city_name": "Barcelona",
"source.geo.continent_name": "Europe",
"source.geo.country_iso_code": "ES",
"source.geo.country_name": "Spain",
"source.geo.location.lat": 41.3891,
"source.geo.location.lon": 2.1611,
"source.geo.region_iso_code": "ES-B",
"source.geo.region_name": "Barcelona",
"source.ip": "79.159.11.115",
"tags": [
"forwarded"
],
"user.domain": "testsiem4.onmicrosoft.com",
"user.email": "root@testsiem4.onmicrosoft.com",
"user.id": "root@testsiem4.onmicrosoft.com",
"user.name": "root"
}
]

0 comments on commit 8323e0c

Please sign in to comment.