Merge remote-tracking branch 'upstream/master' into feature/windows-2008

* upstream/master: [CI] support windows-2012 (elastic#19773) Do not update go.mod during packaging and testing (elastic#19823) Fix typo in ILM warning message (elastic#19819) [Winlogbeat] Remove beta tag from Powershell and Security modules (elastic#19817) feat: move the multibranch pipeline job to the beats repo (elastic#19698) Fix parsing timestamp in Filebeat registry tests (elastic#19796) Add text & flattened fields in aws cloudtrail fileset (elastic#19121)
v1v · Jul 13, 2020 · 2a99271 · 2a99271
2 parents ee0d403 + 29b2d27
commit 2a99271
Show file tree

Hide file tree

Showing 49 changed files with 314 additions and 49 deletions.
diff --git a/.ci/jobs/apm-beats-update.yml b/.ci/jobs/apm-beats-update.yml
@@ -18,7 +18,7 @@
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
         discover-tags: true
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         disable-pr-notifications: true
         notification-context: 'apm-beats-update'
         repo: 'beats'

diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml
@@ -7,7 +7,6 @@
     concurrent: true
     project-type: multibranch
     prune-dead-branches: true
-    number-to-keep: 10
     days-to-keep: 30
     script-path: 'Jenkinsfile'
     triggers: []
@@ -18,12 +17,9 @@
         discover-pr-forks-strategy: 'merge-current'
         discover-pr-forks-trust: 'permission'
         discover-pr-origin: 'merge-current'
-        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+        head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
         discover-tags: true
         notification-context: "beats-ci"
-        property-strategies:
-          all-branches:
-          - suppress-scm-triggering: true
         repo: 'beats'
         repo-owner: 'elastic'
         credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
@@ -32,7 +28,7 @@
         build-strategies:
         - tags:
             ignore-tags-older-than: -1
-            ignore-tags-newer-than: -1
+            ignore-tags-newer-than: 365
         - change-request:
             ignore-target-only-changes: true
         - named-branches:

diff --git a/.ci/jobs/packaging.yml b/.ci/jobs/packaging.yml
@@ -14,7 +14,7 @@
           discover-pr-forks-trust: 'permission'
           discover-pr-origin: 'merge-current'
           discover-tags: true
-          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*)'
+          head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
           disable-pr-notifications: true
           notification-context: 'beats-packaging'
           repo: 'beats'

diff --git a/.ci/windows.groovy b/.ci/windows.groovy
@@ -13,10 +13,10 @@ import groovy.transform.Field
  List of supported windows versions to be tested with
  NOTE:
    - 'windows-10' is too slow
-   - 'windows-2012-r2', 'windows-2008-r2', 'windows-7', 'windows-7-32-bit' are disabled
+   - 'windows-7', 'windows-7-32-bit' are disabled
       since we are working on releasing each windows version incrementally.
 */
-@Field def windowsVersions = ['windows-2019', 'windows-2016', 'windows-2008-r2']
+@Field def windowsVersions = ['windows-2019', 'windows-2016', 'windows-2012-r2', 'windows-2008-r2']
 
 pipeline {
   agent { label 'ubuntu && immutable' }
@@ -349,9 +349,9 @@ def mageTargetWin(String context, String directory, String target, String label)
     log(level: 'INFO', text: "context=${context} directory=${directory} target=${target} os=${label}")
     def immutable = label.equals('windows-7-32-bit') ? 'windows-immutable-32-bit' : 'windows-immutable'
 
-    // NOTE: skip filebeat with windows-2016 since there are some test failures.
-    if (directory.equals('filebeat') && (label.equals('windows-2016') || label.equals('windows-2008-r2'))) {
-      log(level: 'WARN', text: "Skipped stage for the 'filebeat' with '${label}' as long as there are test failures to be analysed.")
+    // NOTE: skip filebeat for windows older than 2016 since there are some test failures.
+    if (directory.equals('filebeat') && (label.equals('windows-2016') || label.equals('windows-2012-r2') || label.equals('windows-2008-r2'))) {
+      log(level: 'WARN', text: "Skipped stage for the 'filebeat' with '${label}' as long as there are test failures to be analysed. See https://github.com/elastic/beats/issues/19641")
     } else {
       node("${immutable} && ${label}"){
         withBeatsEnvWin() {

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -438,6 +438,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346]
 - Improve ECS categorization field mappings in traefik module. {issue}16183[16183] {pull}19379[19379]
 - Improve ECS categorization field mappings in azure module. {issue}16155[16155] {pull}19376[19376]
+- Add text & flattened versions of fields with unknown subfields in aws cloudtrail fileset. {issue}18866[18866] {pull}19121[19121]
 
 *Heartbeat*
 

diff --git a/dev-tools/mage/crossbuild.go b/dev-tools/mage/crossbuild.go
@@ -263,6 +263,7 @@ func (b GolangCrossBuilder) Build() error {
 
 	args = append(args,
 		"--rm",
+		"--env", "GOFLAGS=-mod=readonly",
 		"--env", "MAGEFILE_VERBOSE="+verbose,
 		"--env", "MAGEFILE_TIMEOUT="+EnvOr("MAGEFILE_TIMEOUT", ""),
 		"--env", fmt.Sprintf("SNAPSHOT=%v", Snapshot),

diff --git a/dev-tools/mage/fields.go b/dev-tools/mage/fields.go
@@ -102,6 +102,7 @@ func generateFieldsYAML(baseDir, output string, moduleDirs ...string) error {
 	}
 
 	cmd := []string{"run",
+		"-mod=readonly",
 		filepath.Join(beatsDir, globalFieldsCmdPath),
 		"-es_beats_path", beatsDir,
 		"-beat_path", baseDir,
@@ -127,6 +128,7 @@ func GenerateFieldsGo(fieldsYML, out string) error {
 	}
 
 	cmd := []string{"run",
+		"-mod=readonly",
 		filepath.Join(beatsDir, assetCmdPath),
 		"-pkg", "include",
 		"-in", fieldsYML,

diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go
@@ -236,6 +236,7 @@ func initRunner(tester IntegrationTester, dir string, passInEnv map[string]strin
 	// Create the custom env for the runner.
 	env := map[string]string{
 		insideIntegrationTestEnvVar: "true",
+		"GOFLAGS":                   "-mod=vendor",
 	}
 	for name, value := range passInEnv {
 		env[name] = value

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -1229,6 +1229,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.request_parameters.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.response_elements`*::
 +
 --
@@ -1238,6 +1245,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.response_elements.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.additional_eventdata`*::
 +
 --
@@ -1247,6 +1261,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.additional_eventdata.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.request_id`*::
 +
 --
@@ -1343,6 +1364,13 @@ type: keyword
 
 --
 
+*`aws.cloudtrail.service_event_details.text`*::
++
+--
+type: text
+
+--
+
 *`aws.cloudtrail.shared_event_id`*::
 +
 --
@@ -1401,6 +1429,49 @@ type: boolean
 
 --
 
+[float]
+=== flattened
+
+ES flattened datatype for objects where the subfields aren't known in advance.
+
+
+*`aws.cloudtrail.flattened.additional_eventdata`*::
++
+--
+Additional data about the event that was not part of the request or response.
+
+
+type: flattened
+
+--
+
+*`aws.cloudtrail.flattened.request_parameters`*::
++
+--
+The parameters, if any, that were sent with the request.
+
+type: flattened
+
+--
+
+*`aws.cloudtrail.flattened.response_elements`*::
++
+--
+The response element for actions that make changes (create, update, or delete actions).
+
+type: flattened
+
+--
+
+*`aws.cloudtrail.flattened.service_event_details`*::
++
+--
+Identifies the service event, including what triggered the event and the result.
+
+type: flattened
+
+--
+
 [float]
 === cloudwatch
 

diff --git a/filebeat/tests/system/filebeat.py b/filebeat/tests/system/filebeat.py
@@ -48,7 +48,15 @@ def has_registry(self, name=None, data_path=None):
     def get_registry(self, name=None, data_path=None, filter=None):
         reg = self.access_registry(name, data_path)
         self.wait_until(reg.exists)
-        return reg.load(filter=filter)
+
+        def parse_entry(entry):
+            extra, sec = entry["timestamp"]
+            nsec = extra & 0xFFFFFFFF
+            entry["timestamp"] = sec + (nsec / 1000000000)
+            return entry
+
+        entries = [parse_entry(entry) for entry in reg.load(filter=filter)]
+        return entries
 
     def get_registry_entry_by_path(self, path):
         """

diff --git a/libbeat/idxmgmt/std.go b/libbeat/idxmgmt/std.go
@@ -241,7 +241,7 @@ func (m *indexManager) VerifySetup(loadTemplate, loadILM LoadMode) (bool, string
 	if !ilmComponent.load {
 		warn += "ILM policy and write alias loading not enabled.\n"
 	} else if !ilmComponent.overwrite {
-		warn += "Overwriting ILM policy is disabled. Set `setup.ilm.overwrite:true` for enabling.\n"
+		warn += "Overwriting ILM policy is disabled. Set `setup.ilm.overwrite: true` for enabling.\n"
 	}
 	if !templateComponent.load {
 		warn += "Template loading not enabled.\n"

diff --git a/libbeat/mapping/field.go b/libbeat/mapping/field.go
@@ -138,7 +138,7 @@ func (f *Field) validateType() error {
 		if f.Format != "" {
 			return fmt.Errorf("no format expected for field %s, found: %s", f.Name, f.Format)
 		}
-	case "object", "group", "nested":
+	case "object", "group", "nested", "flattened":
 		// No check for them yet
 	case "":
 		// Module keys, not used as fields

diff --git a/libbeat/tests/system/beat/beat.py b/libbeat/tests/system/beat/beat.py
@@ -571,7 +571,7 @@ def extract_fields(doc_list, name):
                     aliases.extend(subaliases)
                 else:
                     fields.append(newName)
-                    if field.get("type") in ["object", "geo_point"]:
+                    if field.get("type") in ["object", "geo_point", "flattened"]:
                         dictfields.append(newName)
 
                 if field.get("type") == "object" and field.get("object_type") == "histogram":

diff --git a/winlogbeat/docs/modules/powershell.asciidoc b/winlogbeat/docs/modules/powershell.asciidoc
@@ -2,8 +2,6 @@
 [role="xpack"]
 == PowerShell Module
 
-beta[]
-
 The PowerShell module processes event log records from the Microsoft-Windows-PowerShell/Operational and Windows PowerShell logs.
 
 The module has transformations for the following event IDs:

diff --git a/winlogbeat/docs/modules/security.asciidoc b/winlogbeat/docs/modules/security.asciidoc
@@ -2,8 +2,6 @@
 [role="xpack"]
 == Security Module
 
-beta[]
-
 The security module processes event log records from the Security log.
 
 The module has transformations for the following event IDs:

diff --git a/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudtrail/_meta/fields.yml
@@ -90,16 +90,28 @@
       type: keyword
       description: >-
         The parameters, if any, that were sent with the request.
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: response_elements
       type: keyword
       description: >-
         The response element for actions that make changes (create,
         update, or delete actions).
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: additional_eventdata
       type: keyword
       description: >-
         Additional data about the event that was not part of the
         request or response.
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: request_id
       type: keyword
       description: >-
@@ -149,6 +161,10 @@
       description: >-
         Identifies the service event, including what triggered the
         event and the result.
+      multi_fields:
+        - name: text
+          type: text
+          default_field: false
     - name: shared_event_id
       type: keyword
       description: >-
@@ -183,3 +199,27 @@
               description: >-
                 Identifies whether multi factor authentication was
                 used during ConsoleLogin
+    - name: flattened
+      type: group
+      description: >-
+        ES flattened datatype for objects where the subfields aren't known in advance.
+      fields:
+        - name: additional_eventdata
+          type: flattened
+          description: >
+            Additional data about the event that was not part of the
+            request or response.
+        - name: request_parameters
+          type: flattened
+          description: >-
+            The parameters, if any, that were sent with the request.
+        - name: response_elements
+          type: flattened
+          description: >-
+            The response element for actions that make changes (create,
+            update, or delete actions).
+        - name: service_event_details
+          type: flattened
+          description: >-
+            Identifies the service event, including what triggered the
+            event and the result.