From 30e1e18ef47639ae9b02d23dd1c015fbc7e102eb Mon Sep 17 00:00:00 2001 From: James Bebbington Date: Wed, 23 Sep 2020 15:23:25 +1000 Subject: [PATCH] Add new perfcounters package that uses perflib to replace the third_party / pdh package --- go.mod | 1 + go.sum | 6 + .../perfcounters/perfcounter_notwindows.go | 17 ++ .../perfcounters/perfcounter_scraper.go | 184 ++++++++++++++++++ .../perfcounters/perfcounter_scraper_mock.go | 70 +++++++ .../perfcounters/perfcounter_scraper_test.go | 171 ++++++++++++++++ 6 files changed, 449 insertions(+) create mode 100644 receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_notwindows.go create mode 100644 receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper.go create mode 100644 receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_mock.go create mode 100644 receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_test.go diff --git a/go.mod b/go.mod index 205d476d4db..ecef8682906 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/jaegertracing/jaeger v1.19.2 github.com/joshdk/go-junit v0.0.0-20200702055522-6efcf4050909 github.com/jstemmer/go-junit-report v0.9.1 + github.com/leoluk/perflib_exporter v0.1.0 github.com/mjibson/esc v0.2.0 github.com/openzipkin/zipkin-go v0.2.4-0.20200818204336-dc18516bbb4c github.com/orijtech/prometheus-go-metrics-exporter v0.0.5 diff --git a/go.sum b/go.sum index 540ee66753b..185ce8f0c9e 100644 --- a/go.sum +++ b/go.sum @@ -692,6 +692,8 @@ github.com/kyoh86/exportloopref v0.1.7 h1:u+iHuTbkbTS2D/JP7fCuZDo/t3rBVGo3Hf58Rc github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leoluk/perflib_exporter v0.1.0 h1:fXe/mDaf9jR+Zk8FjFlcCSksACuIj2VNN4GyKHmQqtA= +github.com/leoluk/perflib_exporter v0.1.0/go.mod h1:rpV0lYj7lemdTm31t7zpCqYqPnw7xs86f+BaaNBVYFM= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= @@ -884,6 +886,7 @@ github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndr github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prometheus/alertmanager v0.21.0/go.mod h1:h7tJ81NA0VLWvWEayi1QltevFkLF3KxmC/malTcT8Go= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -903,6 +906,7 @@ github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6T github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -913,6 +917,7 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.13.0 h1:vJlpe9wPgDRM1Z+7Wj3zUUjY1nr6/1jNKyl7llliccg= github.com/prometheus/common v0.13.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -1263,6 +1268,7 @@ golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_notwindows.go b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_notwindows.go new file mode 100644 index 00000000000..41148b79c9d --- /dev/null +++ b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_notwindows.go @@ -0,0 +1,17 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build !windows + +package perfcounters diff --git a/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper.go b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper.go new file mode 100644 index 00000000000..fa394f70d93 --- /dev/null +++ b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper.go @@ -0,0 +1,184 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build windows + +package perfcounters + +import ( + "fmt" + "strconv" + "strings" + + "github.com/leoluk/perflib_exporter/perflib" + + "go.opentelemetry.io/collector/internal/processor/filterset" +) + +const totalInstanceName = "_Total" + +type PerfCounterScraper interface { + Initialize(objects ...string) error + Scrape() (PerfDataCollection, error) +} + +type PerfLibScraper struct { + objectIndices string +} + +func (p *PerfLibScraper) Initialize(objects ...string) error { + // "Counter 009" reads perf counter names in English. + // This is always present regardless of the OS language. + nameTable := perflib.QueryNameTable("Counter 009") + + // lookup object indices from name table + objectIndicesMap := map[uint32]struct{}{} + for _, name := range objects { + index := nameTable.LookupIndex(name) + if index == 0 { + return fmt.Errorf("Failed to retrieve perf counter object %q", name) + } + + objectIndicesMap[index] = struct{}{} + } + + // convert to space-separated string + objectIndicesSlice := make([]string, 0, len(objectIndicesMap)) + for k := range objectIndicesMap { + objectIndicesSlice = append(objectIndicesSlice, strconv.Itoa(int(k))) + } + p.objectIndices = strings.Join(objectIndicesSlice, " ") + return nil +} + +func (p *PerfLibScraper) Scrape() (PerfDataCollection, error) { + objects, err := perflib.QueryPerformanceData(p.objectIndices) + if err != nil { + return nil, err + } + + indexed := make(map[string]*perflib.PerfObject) + for _, obj := range objects { + indexed[obj.Name] = obj + } + + return perfDataCollection{perfObject: indexed}, nil +} + +type PerfDataCollection interface { + GetObject(objectName string) (PerfDataObject, error) +} + +type perfDataCollection struct { + perfObject map[string]*perflib.PerfObject +} + +func (p perfDataCollection) GetObject(objectName string) (PerfDataObject, error) { + obj, ok := p.perfObject[objectName] + if !ok { + return nil, fmt.Errorf("Unable to find object %q", objectName) + } + + return perfDataObject{obj}, nil +} + +type PerfDataObject interface { + Filter(includeFS, excludeFS filterset.FilterSet, includeTotal bool) + GetValues(counterNames ...string) ([]*CounterValues, error) +} + +type perfDataObject struct { + *perflib.PerfObject +} + +func (obj perfDataObject) Filter(includeFS, excludeFS filterset.FilterSet, includeTotal bool) { + if includeFS == nil && excludeFS == nil && includeTotal { + return + } + + filteredDevices := make([]*perflib.PerfInstance, 0, len(obj.Instances)) + for _, device := range obj.Instances { + if includeDevice(device.Name, includeFS, excludeFS, includeTotal) { + filteredDevices = append(filteredDevices, device) + } + } + obj.Instances = filteredDevices +} + +func includeDevice(deviceName string, includeFS, excludeFS filterset.FilterSet, includeTotal bool) bool { + if deviceName == totalInstanceName { + return includeTotal + } + + return (includeFS == nil || includeFS.Matches(deviceName)) && + (excludeFS == nil || !excludeFS.Matches(deviceName)) +} + +type CounterValues struct { + InstanceName string + Values map[string]int64 +} + +type counterIndex struct { + index int + name string +} + +func (obj perfDataObject) GetValues(counterNames ...string) ([]*CounterValues, error) { + counterIndices := make([]counterIndex, 0, len(counterNames)) + for idx, counter := range obj.CounterDefs { + // "Base" values give the value of a related counter that pdh.dll uses to compute the derived + // value for this counter. We only care about raw values so ignore base values. See + // https://docs.microsoft.com/en-us/windows/win32/perfctrs/retrieving-counter-data. + if counter.IsBaseValue { + continue + } + + for _, counterName := range counterNames { + if counter.Name == counterName { + counterIndices = append(counterIndices, counterIndex{index: idx, name: counter.Name}) + break + } + } + } + + if len(counterIndices) < len(counterNames) { + return nil, fmt.Errorf("Unable to find counters %q in object %q", missingCounterNames(counterNames, counterIndices), obj.Name) + } + + values := make([]*CounterValues, len(obj.Instances)) + for i, instance := range obj.Instances { + instanceValues := &CounterValues{InstanceName: instance.Name, Values: make(map[string]int64, len(counterIndices))} + for _, counter := range counterIndices { + instanceValues.Values[counter.name] = instance.Counters[counter.index].Value + } + values[i] = instanceValues + } + return values, nil +} + +func missingCounterNames(counterNames []string, counterIndices []counterIndex) []string { + matchedCounters := make(map[string]struct{}, len(counterIndices)) + for _, counter := range counterIndices { + matchedCounters[counter.name] = struct{}{} + } + + counters := make([]string, 0, len(counterNames)-len(matchedCounters)) + for _, counter := range counterNames { + if _, ok := matchedCounters[counter]; !ok { + counters = append(counters, counter) + } + } + return counters +} diff --git a/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_mock.go b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_mock.go new file mode 100644 index 00000000000..b9ea898d8df --- /dev/null +++ b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_mock.go @@ -0,0 +1,70 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build windows + +package perfcounters + +import ( + "go.opentelemetry.io/collector/internal/processor/filterset" +) + +// MockPerfCounterScraperError returns the supplied errors when Scrape, GetObject, +// or GetValues are called. + +type MockPerfCounterScraperError struct { + scrapeErr error + getObjectErr error + getValuesErr error +} + +func NewMockPerfCounterScraperError(scrapeErr, getObjectErr, getValuesErr error) *MockPerfCounterScraperError { + return &MockPerfCounterScraperError{scrapeErr: scrapeErr, getObjectErr: getObjectErr, getValuesErr: getValuesErr} +} + +func (p *MockPerfCounterScraperError) Initialize(objects ...string) error { + return nil +} + +func (p *MockPerfCounterScraperError) Scrape() (PerfDataCollection, error) { + if p.scrapeErr != nil { + return nil, p.scrapeErr + } + + return mockPerfDataCollectionError{getObjectErr: p.getObjectErr, getValuesErr: p.getValuesErr}, nil +} + +type mockPerfDataCollectionError struct { + getObjectErr error + getValuesErr error +} + +func (p mockPerfDataCollectionError) GetObject(objectName string) (PerfDataObject, error) { + if p.getObjectErr != nil { + return nil, p.getObjectErr + } + + return mockPerfDataObjectError{getValuesErr: p.getValuesErr}, nil +} + +type mockPerfDataObjectError struct { + getValuesErr error +} + +func (obj mockPerfDataObjectError) Filter(includeFS, excludeFS filterset.FilterSet, includeTotal bool) { +} + +func (obj mockPerfDataObjectError) GetValues(counterNames ...string) ([]*CounterValues, error) { + return nil, obj.getValuesErr +} diff --git a/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_test.go b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_test.go new file mode 100644 index 00000000000..c699f0e99cc --- /dev/null +++ b/receiver/hostmetricsreceiver/internal/perfcounters/perfcounter_scraper_test.go @@ -0,0 +1,171 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build windows + +package perfcounters + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "go.opentelemetry.io/collector/internal/processor/filterset" +) + +func Test_PerfCounterScraper(t *testing.T) { + type testCase struct { + name string + // NewPerfCounter + objects []string + newErr string + expectIndices []string + // Filter + includeFS filterset.FilterSet + excludeFS filterset.FilterSet + includeTotal bool + // GetObject + getObject string + getObjectErr string + // GetCounterValues + getCounters []string + getCountersErr string + expectedInstanceNames []string + excludedInstanceNames []string + expectedMinimumInstances int + } + + excludedCommonDrives := []string{"C:"} + excludeCommonDriveFilterSet, err := filterset.CreateFilterSet(excludedCommonDrives, &filterset.Config{MatchType: filterset.Strict}) + require.NoError(t, err) + + testCases := []testCase{ + { + name: "Standard", + objects: []string{"Memory"}, + expectIndices: []string{"4"}, + getObject: "Memory", + getCounters: []string{"Committed Bytes"}, + expectedInstanceNames: []string{""}, + }, + { + name: "Multiple Objects & Values", + objects: []string{"Memory", "LogicalDisk"}, + expectIndices: []string{"4", "236"}, + getObject: "LogicalDisk", + getCounters: []string{"Disk Reads/sec", "Disk Writes/sec"}, + expectedMinimumInstances: 1, + }, + { + name: "Filtered", + objects: []string{"LogicalDisk"}, + expectIndices: []string{"236"}, + excludeFS: excludeCommonDriveFilterSet, + includeTotal: true, + getObject: "LogicalDisk", + getCounters: []string{"Disk Reads/sec"}, + excludedInstanceNames: excludedCommonDrives, + }, + { + name: "New Error", + objects: []string{"Memory", "Invalid Object 1", "Invalid Object 2"}, + newErr: `Failed to retrieve perf counter object "Invalid Object 1"`, + }, + { + name: "Get Object Error", + objects: []string{"Memory"}, + expectIndices: []string{"4"}, + getObject: "Invalid Object 1", + getObjectErr: `Unable to find object "Invalid Object 1"`, + }, + { + name: "Get Values Error", + objects: []string{"Memory"}, + expectIndices: []string{"4"}, + getObject: "Memory", + getCounters: []string{"Committed Bytes", "Invalid Counter 1", "Invalid Counter 2"}, + getCountersErr: `Unable to find counters ["Invalid Counter 1" "Invalid Counter 2"] in object "Memory"`, + }, + } + + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + s := &PerfLibScraper{} + err := s.Initialize(test.objects...) + if test.newErr != "" { + assert.EqualError(t, err, test.newErr) + return + } + require.NoError(t, err, "Failed to create new perf counter scraper: %v", err) + + assert.ElementsMatch(t, test.expectIndices, strings.Split(s.objectIndices, " ")) + + c, err := s.Scrape() + require.NoError(t, err, "Failed to scrape data: %v", err) + + p, err := c.GetObject(test.getObject) + if test.getObjectErr != "" { + assert.EqualError(t, err, test.getObjectErr) + return + } + require.NoError(t, err, "Failed to get object: %v", err) + + p.Filter(test.includeFS, test.excludeFS, test.includeTotal) + + counterValues, err := p.GetValues(test.getCounters...) + if test.getCountersErr != "" { + assert.EqualError(t, err, test.getCountersErr) + return + } + require.NoError(t, err, "Failed to get counter: %v", err) + + assert.GreaterOrEqual(t, len(counterValues), test.expectedMinimumInstances) + + if len(test.expectedInstanceNames) > 0 { + for _, expectedName := range test.expectedInstanceNames { + var gotName bool + for _, cv := range counterValues { + if cv.InstanceName == expectedName { + gotName = true + break + } + } + assert.Truef(t, gotName, "Expected Instance %q was not returned", expectedName) + } + } + + if len(test.excludedInstanceNames) > 0 { + for _, excludedName := range test.excludedInstanceNames { + for _, cv := range counterValues { + if cv.InstanceName == excludedName { + assert.Fail(t, "", "Excluded Instance %q was returned", excludedName) + break + } + } + } + } + + var includesTotal bool + for _, cv := range counterValues { + if cv.InstanceName == "_Total" { + includesTotal = true + break + } + } + assert.Equalf(t, test.includeTotal, includesTotal, "_Total was returned: %v (expected the opposite)", test.includeTotal, includesTotal) + }) + } +}