Merge branch 'master' into syslog-docs

CHANGELOG-developer.next.asciidoc

@@ -53,6 +53,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove global ACK handler support via `SetACKHandler` from publisher pipeline. {pull}19632[19632]
 - Make implementing `Close` required for `reader.Reader` interfaces. {pull}20455[20455]
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
+- Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -161,6 +161,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Fix events being dropped if they contain a floating point value of NaN or Inf. {pull}25051[25051]
 - Fix templates being overwritten if there was an error when check for the template existance. {pull}24332[24332]
 - Fix Kubernetes autodiscovery provider to correctly handle pod states and avoid missing event data {pull}17223[17223]
 - Fix `add_cloud_metadata` to better support modifying sub-fields with other processors. {pull}13808[13808]

NOTICE.txt

@@ -7454,11 +7454,11 @@ Contents of probable licence file $GOMODCACHE/github.com/elastic/go-seccomp-bpf@
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-structform
-Version: v0.0.7
+Version: v0.0.8
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.7/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-structform@v0.0.8/LICENSE:
 
                                  Apache License
                            Version 2.0, January 2004
@@ -14556,11 +14556,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/stretchr/testify
-Version: v1.6.1
+Version: v1.7.0
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.6.1/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.7.0/LICENSE:
 
 MIT License
 

go.mod

@@ -68,7 +68,7 @@ require (
 	github.com/elastic/go-lumber v0.1.0
 	github.com/elastic/go-perf v0.0.0-20191212140718-9c656876f595
 	github.com/elastic/go-seccomp-bpf v1.1.0
-	github.com/elastic/go-structform v0.0.7
+	github.com/elastic/go-structform v0.0.8
 	github.com/elastic/go-sysinfo v1.6.0
 	github.com/elastic/go-txfile v0.0.7
 	github.com/elastic/go-ucfg v0.8.3
@@ -147,7 +147,7 @@ require (
 	github.com/spf13/cobra v0.0.5
 	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/stretchr/testify v1.6.1
+	github.com/stretchr/testify v1.7.0
 	github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b
 	github.com/tsg/gopacket v0.0.0-20200626092518-2ab8e397a786
 	github.com/ugorji/go/codec v1.1.8

go.sum

@@ -296,8 +296,8 @@ github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f h1:Fvsq
 github.com/elastic/go-plugins-helpers v0.0.0-20200207104224-bdf17607b79f/go.mod h1:OPGqFNdTS34kMReS5hPFtBhD9J8itmSDurs1ix2wx7c=
 github.com/elastic/go-seccomp-bpf v1.1.0 h1:jUzzDc6LyCtdolZdvL/26dad6rZ9vsc7xZ2eadKECAU=
 github.com/elastic/go-seccomp-bpf v1.1.0/go.mod h1:l+89Vy5BzjVcaX8USZRMOwmwwDScE+vxCFzzvQwN7T8=
-github.com/elastic/go-structform v0.0.7 h1:ihszOJQryNuIIHE2ZgsbiDq+agKO6V4yK0JYAI3tjzc=
-github.com/elastic/go-structform v0.0.7/go.mod h1:QrMyP3oM9Sjk92EVGLgRaL2lKt0Qx7ZNDRWDxB6khVs=
+github.com/elastic/go-structform v0.0.8 h1:U0qnb9Zqig7w+FhF+sLI3VZPPi/+2aJ0bIEW6R1z6Tk=
+github.com/elastic/go-structform v0.0.8/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
 github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
 github.com/elastic/go-sysinfo v1.6.0 h1:u0QbU8eWSwKRPcFQancnSY4Zi0COksCJXkUgPHxE5Tw=
 github.com/elastic/go-sysinfo v1.6.0/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
@@ -768,6 +768,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 h1:zLV6q4e8Jv9EHjNg/iHfzwDkCve6Ua5jCygptrtXHvI=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b h1:X/8hkb4rQq3+QuOxpJK7gWmAXmZucF0EI1s1BfBLq6U=
 github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b/go.mod h1:jAqhj/JBVC1PwcLTWd6rjQyGyItxxrhpiBl8LSuAGmw=

libbeat/outputs/codec/json/json.go

@@ -73,6 +73,7 @@ func New(version string, config Config) *Encoder {
 func (e *Encoder) reset() {
 	visitor := json.NewVisitor(&e.buf)
 	visitor.SetEscapeHTML(e.config.EscapeHTML)
+	visitor.SetIgnoreInvalidFloat(true)
 
 	var err error
 

libbeat/outputs/codec/json/json_test.go

@@ -18,6 +18,7 @@
 package json
 
 import (
+	"math"
 	"testing"
 	"time"
 
@@ -52,27 +53,31 @@ func TestJsonCodec(t *testing.T) {
   "msg": "message"
 }`,
 		},
-		"html escaping enabled": testCase{
+		"html escaping enabled": {
 			config:   Config{EscapeHTML: true},
 			in:       common.MapStr{"msg": "<hello>world</hello>"},
 			expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"\u003chello\u003eworld\u003c/hello\u003e"}`,
 		},
-		"html escaping disabled": testCase{
+		"html escaping disabled": {
 			config:   Config{EscapeHTML: false},
 			in:       common.MapStr{"msg": "<hello>world</hello>"},
 			expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"<hello>world</hello>"}`,
 		},
-		"UTC timezone offset": testCase{
+		"UTC timezone offset": {
 			config:   Config{LocalTime: true},
 			in:       common.MapStr{"msg": "message"},
 			expected: `{"@timestamp":"0001-01-01T00:00:00.000+00:00","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
 		},
-		"PST timezone offset": testCase{
+		"PST timezone offset": {
 			config:   Config{LocalTime: true},
 			ts:       time.Time{}.In(time.FixedZone("PST", -8*60*60)),
 			in:       common.MapStr{"msg": "message"},
 			expected: `{"@timestamp":"0000-12-31T16:00:00.000-08:00","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
 		},
+		"float undefined values": {
+			in:       common.MapStr{"nan": math.NaN()},
+			expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"nan":null}`,
+		},
 	}
 
 	for name, test := range cases {

metricbeat/helper/prometheus/ptest/ptest.go

@@ -30,7 +30,6 @@ import (
 
 	"github.com/stretchr/testify/assert"
 
-	"github.com/elastic/beats/v7/libbeat/common"
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
 	"github.com/elastic/beats/v7/metricbeat/mb/testing/flags"
@@ -45,87 +44,6 @@ type TestCases []struct {
 	ExpectedFile string
 }
 
-// TestMetricSetEventsFetcher goes over the given TestCases and ensures that source Prometheus metrics gets converted
-// into the expected events when passed by the given metricset.
-// If -data flag is passed, the expected JSON file will be updated with the result
-func TestMetricSetEventsFetcher(t *testing.T, module, metricset string, cases TestCases) {
-	for _, test := range cases {
-		t.Logf("Testing %s file\n", test.MetricsFile)
-
-		file, err := os.Open(test.MetricsFile)
-		assert.NoError(t, err, "cannot open test file "+test.MetricsFile)
-
-		body, err := ioutil.ReadAll(file)
-		assert.NoError(t, err, "cannot read test file "+test.MetricsFile)
-
-		server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(200)
-			w.Header().Set("Content-Type", "text/plain; charset=ISO-8859-1")
-			w.Write([]byte(body))
-		}))
-
-		server.Start()
-		defer server.Close()
-
-		config := map[string]interface{}{
-			"module":     module,
-			"metricsets": []string{metricset},
-			"hosts":      []string{server.URL},
-		}
-
-		f := mbtest.NewEventsFetcher(t, config)
-		events, err := f.Fetch()
-		assert.Nil(t, err, "Errors while fetching metrics")
-
-		if *flags.DataFlag {
-			sort.SliceStable(events, func(i, j int) bool {
-				h1, _ := hashstructure.Hash(events[i], nil)
-				h2, _ := hashstructure.Hash(events[j], nil)
-				return h1 < h2
-			})
-			eventsJSON, _ := json.MarshalIndent(events, "", "\t")
-			err = ioutil.WriteFile(test.ExpectedFile, eventsJSON, 0644)
-			assert.NoError(t, err)
-		}
-
-		// Read expected events from reference file
-		expected, err := ioutil.ReadFile(test.ExpectedFile)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		var expectedEvents []common.MapStr
-		err = json.Unmarshal(expected, &expectedEvents)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		for _, event := range events {
-			// ensure the event is in expected list
-			found := -1
-			for i, expectedEvent := range expectedEvents {
-				if event.String() == expectedEvent.String() {
-					found = i
-					break
-				}
-			}
-			if found > -1 {
-				expectedEvents = append(expectedEvents[:found], expectedEvents[found+1:]...)
-			} else {
-				t.Errorf("Event was not expected: %+v", event)
-			}
-		}
-
-		if len(expectedEvents) > 0 {
-			t.Error("Some events were missing:")
-			for _, e := range expectedEvents {
-				t.Error(e)
-			}
-			t.Fatal()
-		}
-	}
-}
-
 // TestMetricSet goes over the given TestCases and ensures that source Prometheus metrics gets converted into the expected
 // events when passed by the given metricset.
 // If -data flag is passed, the expected JSON file will be updated with the result

metricbeat/mb/builders.go

@@ -217,14 +217,6 @@ func mustHaveModule(ms MetricSet, base BaseMetricSet) error {
 // of them.
 func mustImplementFetcher(ms MetricSet) error {
 	var ifcs []string
-	if _, ok := ms.(EventFetcher); ok {
-		ifcs = append(ifcs, "EventFetcher")
-	}
-
-	if _, ok := ms.(EventsFetcher); ok {
-		ifcs = append(ifcs, "EventsFetcher")
-	}
-
 	if _, ok := ms.(ReportingMetricSet); ok {
 		ifcs = append(ifcs, "ReportingMetricSet")
 	}
@@ -256,7 +248,7 @@ func mustImplementFetcher(ms MetricSet) error {
 	switch len(ifcs) {
 	case 0:
 		return fmt.Errorf("MetricSet '%s/%s' does not implement an event "+
-			"producing interface (EventFetcher, EventsFetcher, "+
+			"producing interface ("+
 			"ReportingMetricSet, ReportingMetricSetV2, ReportingMetricSetV2Error, ReportingMetricSetV2WithContext"+
 			"PushMetricSet, PushMetricSetV2, or PushMetricSetV2WithContext)",
 			ms.Module().Name(), ms.Name())

metricbeat/mb/mb.go

@@ -133,8 +133,7 @@ func (m *BaseModule) WithConfig(config common.Config) (*BaseModule, error) {
 // MetricSet interfaces
 
 // MetricSet is the common interface for all MetricSet implementations. In
-// addition to this interface, all MetricSets must implement either
-// EventFetcher or EventsFetcher (but not both).
+// addition to this interface, all MetricSets must implement a fetcher interface.
 type MetricSet interface {
 	ID() string     // Unique ID identifying a running MetricSet.
 	Name() string   // Name returns the name of the MetricSet.
@@ -154,20 +153,6 @@ type Closer interface {
 	Close() error
 }
 
-// EventFetcher is a MetricSet that returns a single event when collecting data.
-// Use ReportingMetricSet for new MetricSet implementations.
-type EventFetcher interface {
-	MetricSet
-	Fetch() (common.MapStr, error)
-}
-
-// EventsFetcher is a MetricSet that returns a multiple events when collecting
-// data. Use ReportingMetricSet for new MetricSet implementations.
-type EventsFetcher interface {
-	MetricSet
-	Fetch() ([]common.MapStr, error)
-}
-
 // Reporter is used by a MetricSet to report events, errors, or errors with
 // metadata. The methods return false if and only if publishing failed because
 // the MetricSet is being closed.

metricbeat/mb/mb_test.go

@@ -30,6 +30,8 @@ import (
 	"github.com/elastic/beats/v7/libbeat/common"
 )
 
+// Reporting V2 MetricSet
+
 type testModule struct {
 	BaseModule
 	hostParser func(string) (HostData, error)
@@ -39,25 +41,11 @@ func (m testModule) ParseHost(host string) (HostData, error) {
 	return m.hostParser(host)
 }
 
-// EventFetcher
-
 type testMetricSet struct {
 	BaseMetricSet
 }
 
-func (m *testMetricSet) Fetch() (common.MapStr, error) {
-	return nil, nil
-}
-
-// EventsFetcher
-
-type testMetricSetEventsFetcher struct {
-	BaseMetricSet
-}
-
-func (m *testMetricSetEventsFetcher) Fetch() ([]common.MapStr, error) {
-	return nil, nil
-}
+func (m *testMetricSet) Fetch(reporter ReporterV2) {}
 
 // ReportingFetcher
 
@@ -259,25 +247,7 @@ func TestNewModulesMetricSetTypes(t *testing.T) {
 		return &testMetricSet{base}, nil
 	}
 
-	name := "EventFetcher"
-	if err := r.AddMetricSet(moduleName, name, factory); err != nil {
-		t.Fatal(err)
-	}
-
-	t.Run(name+" MetricSet", func(t *testing.T) {
-		ms := newTestMetricSet(t, r, map[string]interface{}{
-			"module":     moduleName,
-			"metricsets": []string{name},
-		})
-		_, ok := ms.(EventFetcher)
-		assert.True(t, ok, name+" not implemented")
-	})
-
-	factory = func(base BaseMetricSet) (MetricSet, error) {
-		return &testMetricSetEventsFetcher{base}, nil
-	}
-
-	name = "EventsFetcher"
+	name := "ReportingMetricSetV2"
 	if err := r.AddMetricSet(moduleName, name, factory); err != nil {
 		t.Fatal(err)
 	}
@@ -287,7 +257,7 @@ func TestNewModulesMetricSetTypes(t *testing.T) {
 			"module":     moduleName,
 			"metricsets": []string{name},
 		})
-		_, ok := ms.(EventsFetcher)
+		_, ok := ms.(ReportingMetricSetV2)
 		assert.True(t, ok, name+" not implemented")
 	})
 

metricbeat/mb/module/example_test.go

@@ -40,7 +40,7 @@ func ExampleWrapper() {
 	// Build a configuration object.
 	config, err := common.NewConfigFrom(map[string]interface{}{
 		"module":     moduleName,
-		"metricsets": []string{eventFetcherName},
+		"metricsets": []string{reportingFetcherName},
 	})
 	if err != nil {
 		fmt.Println("Error:", err)
@@ -91,17 +91,17 @@ func ExampleWrapper() {
 	//   },
 	//   "@timestamp": "2016-05-10T23:27:58.485Z",
 	//   "event": {
-	//     "dataset": "fake.eventfetcher",
+	//     "dataset": "fake.reportingfetcher",
 	//     "duration": 111,
 	//     "module": "fake"
 	//   },
 	//   "fake": {
-	//     "eventfetcher": {
+	//     "reportingfetcher": {
 	//       "metric": 1
 	//     }
 	//   },
 	//   "metricset": {
-	//     "name": "eventfetcher",
+	//     "name": "reportingfetcher",
 	//     "period": 10000
 	//   },
 	//   "service": {
@@ -120,7 +120,7 @@ func ExampleRunner() {
 
 	config, err := common.NewConfigFrom(map[string]interface{}{
 		"module":     moduleName,
-		"metricsets": []string{eventFetcherName},
+		"metricsets": []string{reportingFetcherName},
 	})
 	if err != nil {
 		return