Cherry-pick elastic#18948 to 7.7: Filebeat: Fix o365 module issues (e…

…lastic#19063)

* Filebeat: Fix o365 module issues (elastic#18948)

- Fix scary data-loss warning on startup
- Avoid API errors being processed by the JS pipeline
- Fix dissect error about overiding client.port
- Fix module passing API settings to the input
- Document max_period using the right units

(cherry picked from commit b99a73c)

CHANGELOG.next.asciidoc

@@ -94,6 +94,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847]
 - Fixing `ingress_controller.` fields to be of type keyword instead of text. {issue}17834[17834]
 - Fixed typo in log message. {pull}17897[17897]
+- Fix `o365` module ignoring `var.api` settings. {pull}18948[18948]
 
 *Heartbeat*
 

x-pack/filebeat/filebeat.reference.yml

@@ -794,7 +794,7 @@ filebeat.modules:
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m
 

x-pack/filebeat/input/o365audit/state.go

@@ -114,10 +114,7 @@ func (s *stateStorage) Load(key stream) (cursor, error) {
 	}
 	cur, err := s.persister.Load(key)
 	if err != nil {
-		if err != errStateNotFound {
-			return cur, err
-		}
-		cur = newCursor(key, time.Time{})
+		return newCursor(key, time.Time{}), err
 	}
 	return cur, s.saveUnsafe(cur)
 }

x-pack/filebeat/input/o365audit/state_test.go

@@ -21,18 +21,14 @@ func TestNoopState(t *testing.T) {
 	t.Run("new state", func(t *testing.T) {
 		st := newStateStorage(noopPersister{})
 		cur, err := st.Load(myStream)
-		if !assert.NoError(t, err) {
-			t.Fatal(err)
-		}
+		assert.Equal(t, errStateNotFound, err)
 		empty := newCursor(myStream, time.Time{})
 		assert.Equal(t, empty, cur)
 	})
 	t.Run("update state", func(t *testing.T) {
 		st := newStateStorage(noopPersister{})
 		cur, err := st.Load(myStream)
-		if !assert.NoError(t, err) {
-			t.Fatal(err)
-		}
+		assert.Equal(t, errStateNotFound, err)
 		advanced := cur.TryAdvance(content{
 			Type:       tn,
 			ID:         "1234",

x-pack/filebeat/module/o365/_meta/config.yml

@@ -40,6 +40,6 @@
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m

x-pack/filebeat/module/o365/audit/config/input.yml

@@ -21,7 +21,7 @@ content_type:
 {{ if .api }}
 api:
 {{ range $k, $v := .api }}
- - {{ $k }}: {{ $v -}}
+  {{ $k }}: {{ $v -}}
 {{ end }}
 {{ end }}
 
@@ -49,6 +49,7 @@ processors:
         - 2006-01-02T15:04:05
 {{ end }}
   - script:
+      when.has_fields: ['o365audit']
       lang: javascript
       id: o365audit_script
       file: ${path.home}/module/o365/audit/config/pipeline.js

x-pack/filebeat/module/o365/audit/config/pipeline.js

@@ -729,7 +729,10 @@ function AuditProcessor(tenant_names, debug) {
         tokenizer: '[%{_ip}]:%{port}',
         field: 'client.address',
         target_prefix: 'client',
-        'when.contains.client.address': ']:',
+        'when.and': [
+            {'not.has_fields': ['client._ip', 'client.port']},
+            {'contains.client.address': ']:'},
+        ],
     }));
     builder.Add("extractClientIPv4Port", new processor.Dissect({
         tokenizer: '%{_ip}:%{port}',

x-pack/filebeat/modules.d/o365.yml.disabled

@@ -43,6 +43,6 @@
     #   authentication_endpoint: "https://login.microsoftonline.us/"
     #   resource: "https://manage.office365.us"
     #
-    #   max_retention: 7d
+    #   max_retention: 168h
     #   max_requests_per_minute: 2000
     #   poll_interval: 3m