From f34aaa61092fe7e3c6618fdb0b0d16a68a291ff7 Mon Sep 17 00:00:00 2001 From: Conall O'Brien Date: Mon, 11 Sep 2023 05:33:21 +0100 Subject: [PATCH] Add ZFS freebsd per dataset stats (#2753) * Rename parsePoolObjsetFile to parseLinuxPoolObjsetFile to better reflect it's scope * Create a new parseFreeBSDPoolObjsetStats function, to generate a list of per pool metrics to be queried via sysctl --------- Signed-off-by: Conall O'Brien --- collector/exec_bsd.go | 6 ++++ collector/memory_bsd.go | 12 +++++++ collector/netisr_freebsd.go | 6 ++++ collector/sysctl_bsd.go | 3 ++ collector/zfs_freebsd.go | 70 +++++++++++++++++++++++++++++++++++++ collector/zfs_linux.go | 4 +-- collector/zfs_linux_test.go | 2 +- 7 files changed, 100 insertions(+), 3 deletions(-) diff --git a/collector/exec_bsd.go b/collector/exec_bsd.go index 87cb3ceeb4..7ae5e2ee32 100644 --- a/collector/exec_bsd.go +++ b/collector/exec_bsd.go @@ -49,31 +49,37 @@ func NewExecCollector(logger log.Logger) (Collector, error) { name: "exec_context_switches_total", description: "Context switches since system boot. Resets at architecture unsigned integer.", mib: "vm.stats.sys.v_swtch", + labels: nil, }, { name: "exec_traps_total", description: "Traps since system boot. Resets at architecture unsigned integer.", mib: "vm.stats.sys.v_trap", + labels: nil, }, { name: "exec_system_calls_total", description: "System calls since system boot. Resets at architecture unsigned integer.", mib: "vm.stats.sys.v_syscall", }, + labels: nil, { name: "exec_device_interrupts_total", description: "Device interrupts since system boot. Resets at architecture unsigned integer.", mib: "vm.stats.sys.v_intr", + labels: nil, }, { name: "exec_software_interrupts_total", description: "Software interrupts since system boot. Resets at architecture unsigned integer.", mib: "vm.stats.sys.v_soft", + labels: nil, }, { name: "exec_forks_total", description: "Number of fork() calls since system boot. Resets at architecture unsigned integer.", mib: "vm.stats.vm.v_forks", + labels: nil, }, }, logger: logger, diff --git a/collector/memory_bsd.go b/collector/memory_bsd.go index 6af9d8aaa4..6f16ac896d 100644 --- a/collector/memory_bsd.go +++ b/collector/memory_bsd.go @@ -69,18 +69,21 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) { description: "Recently used by userland", mib: "vm.stats.vm.v_active_count", conversion: fromPage, + labels: nil, }, { name: "inactive_bytes", description: "Not recently used by userland", mib: "vm.stats.vm.v_inactive_count", conversion: fromPage, + labels: nil, }, { name: "wired_bytes", description: "Locked in memory by kernel, mlock, etc", mib: "vm.stats.vm.v_wire_count", conversion: fromPage, + labels: nil, }, { name: "user_wired_bytes", @@ -88,42 +91,49 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) { mib: "vm.stats.vm.v_user_wire_count", conversion: fromPage, dataType: bsdSysctlTypeCLong, + labels: nil, }, { name: "cache_bytes", description: "Almost free, backed by swap or files, available for re-allocation", mib: "vm.stats.vm.v_cache_count", conversion: fromPage, + labels: nil, }, { name: "buffer_bytes", description: "Disk IO Cache entries for non ZFS filesystems, only usable by kernel", mib: "vfs.bufspace", dataType: bsdSysctlTypeCLong, + labels: nil, }, { name: "free_bytes", description: "Unallocated, available for allocation", mib: "vm.stats.vm.v_free_count", conversion: fromPage, + labels: nil, }, { name: "laundry_bytes", description: "Dirty not recently used by userland", mib: "vm.stats.vm.v_laundry_count", conversion: fromPage, + labels: nil, }, { name: "size_bytes", description: "Total physical memory size", mib: "vm.stats.vm.v_page_count", conversion: fromPage, + labels: nil, }, { name: "swap_size_bytes", description: "Total swap memory size", mib: mibSwapTotal, dataType: bsdSysctlTypeUint64, + labels: nil, }, // Descriptions via: top(1) { @@ -132,6 +142,7 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) { mib: "vm.stats.vm.v_swappgsin", valueType: prometheus.CounterValue, conversion: fromPage, + labels: nil, }, { name: "swap_out_bytes_total", @@ -139,6 +150,7 @@ func NewMemoryCollector(logger log.Logger) (Collector, error) { mib: "vm.stats.vm.v_swappgsout", valueType: prometheus.CounterValue, conversion: fromPage, + labels: nil, }, }, }, nil diff --git a/collector/netisr_freebsd.go b/collector/netisr_freebsd.go index 442bcdc6a7..43991f3ab9 100644 --- a/collector/netisr_freebsd.go +++ b/collector/netisr_freebsd.go @@ -45,6 +45,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) { mib: "net.isr.numthreads", dataType: bsdSysctlTypeUint32, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "maxprot", @@ -52,6 +53,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) { mib: "net.isr.maxprot", dataType: bsdSysctlTypeUint32, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "defaultqlimit", @@ -59,6 +61,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) { mib: "net.isr.defaultqlimit", dataType: bsdSysctlTypeUint32, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "maxqlimit", @@ -66,6 +69,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) { mib: "net.isr.maxqlimit", dataType: bsdSysctlTypeUint32, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "bindthreads", @@ -73,6 +77,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) { mib: "net.isr.bindthreads", dataType: bsdSysctlTypeUint32, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "maxthreads", @@ -80,6 +85,7 @@ func NewNetisrCollector(logger log.Logger) (Collector, error) { mib: "net.isr.maxthreads", dataType: bsdSysctlTypeUint32, valueType: prometheus.GaugeValue, + labels: nil, }, }, logger: logger, diff --git a/collector/sysctl_bsd.go b/collector/sysctl_bsd.go index 2ab248ed2e..df5c9f93f1 100644 --- a/collector/sysctl_bsd.go +++ b/collector/sysctl_bsd.go @@ -59,6 +59,9 @@ type bsdSysctl struct { // Post-retrieval conversion hooks conversion func(float64) float64 + + // Prometheus labels + labels prometheus.Labels } func (b bsdSysctl) Value() (float64, error) { diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go index 2418b229d0..2b3948a50a 100644 --- a/collector/zfs_freebsd.go +++ b/collector/zfs_freebsd.go @@ -21,6 +21,8 @@ import ( "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" + + "golang.org/x/sys/unix" ) type zfsCollector struct { @@ -45,6 +47,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.abdstats.linear_cnt", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "abdstats_linear_data_bytes", @@ -52,6 +55,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.abdstats.linear_data_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "abdstats_scatter_chunk_waste_bytes", @@ -59,6 +63,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.abdstats.scatter_chunk_waste", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "abdstats_scatter_count_total", @@ -66,6 +71,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.abdstats.scatter_cnt", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "abdstats_scatter_data_bytes", @@ -73,6 +79,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.abdstats.scatter_data_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "abdstats_struct_bytes", @@ -80,6 +87,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.abdstats.struct_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_anon_bytes", @@ -87,6 +95,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.anon_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_c_bytes", @@ -94,6 +103,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.c", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_c_max_bytes", @@ -101,6 +111,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.c_max", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_c_min_bytes", @@ -108,6 +119,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.c_min", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_data_bytes", @@ -115,6 +127,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.data_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_demand_data_hits_total", @@ -122,6 +135,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.demand_data_hits", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_demand_data_misses_total", @@ -129,6 +143,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.demand_data_misses", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_demand_metadata_hits_total", @@ -136,6 +151,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.demand_metadata_hits", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_demand_metadata_misses_total", @@ -143,6 +159,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.demand_metadata_misses", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_hdr_bytes", @@ -150,6 +167,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.hdr_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_hits_total", @@ -157,6 +175,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.hits", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_misses_total", @@ -164,6 +183,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.misses", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_mfu_ghost_hits_total", @@ -171,6 +191,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.mfu_ghost_hits", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_mfu_ghost_size", @@ -178,6 +199,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.mfu_ghost_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_mfu_bytes", @@ -185,6 +207,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.mfu_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_mru_ghost_hits_total", @@ -192,6 +215,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.mru_ghost_hits", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "arcstats_mru_ghost_bytes", @@ -199,6 +223,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.mru_ghost_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_mru_bytes", @@ -206,6 +231,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.mru_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_other_bytes", @@ -213,6 +239,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.other_size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_p_bytes", @@ -220,6 +247,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.p", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "arcstats_size_bytes", @@ -227,6 +255,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.arcstats.size", dataType: bsdSysctlTypeUint64, valueType: prometheus.GaugeValue, + labels: nil, }, { name: "zfetchstats_hits_total", @@ -234,6 +263,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.zfetchstats.hits", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, { name: "zfetchstats_misses_total", @@ -241,6 +271,7 @@ func NewZfsCollector(logger log.Logger) (Collector, error) { mib: "kstat.zfs.misc.zfetchstats.misses", dataType: bsdSysctlTypeUint64, valueType: prometheus.CounterValue, + labels: nil, }, }, logger: logger, @@ -264,3 +295,42 @@ func (c *zfsCollector) Update(ch chan<- prometheus.Metric) error { return nil } + +func (c *zfsCollector) parseFreeBSDPoolObjsetStats() error { + + sysCtlMetrics := []string{ + "nunlinked", "nunlinks", "nread", "reads", "nwritten", "writes", + } + zfsPoolMibPrefix := "kstat.zfs.pool.dataset" + zfsDatasetsNames := []string{} + + zfsDatasets, err := unix.Sysctl(zfsPoolMibPrefix) + if err != nil { + return fmt.Errorf("couldn't get sysctl: %w", err) + } + + for dataset, _ := range zfsDatasets { + if strings.HasSuffix(dataset, ".dataset_name") { + zfsDatasetNames = append(zfsDatasetNames, strings.SplitAfter(dataset, ".")[3]) + } + } + + for zpoolDataset := range zfsDatasetsNames { + zfsDatasetLabels := map[string]string{ + "dataset": zpoolDataset, + "zpool": strings.SplitAfter(zpoolDataset, "/")[0], + } + for metric := range sysCtlMetrics { + c.sysctls = append(c.sysctls, bsdSysctl{ + name: fmt.SprintF("node_zfs_zpool_dataset_%s", metric), + description: fmt.SprintF("node_zfs_zpool_dataset_%s", metric), + mib: fmt.Sprintf("%s.%s.%s", zfsPoolMibPrefix, poolObj, metric), + dataType: bsdSysctlTypeUint64, + valueType: prometheus.CounterValue, + labels: zfsDatasetLabels, + }) + } + } + + return nil +} diff --git a/collector/zfs_linux.go b/collector/zfs_linux.go index ec195b3d62..e7be4a9836 100644 --- a/collector/zfs_linux.go +++ b/collector/zfs_linux.go @@ -104,7 +104,7 @@ func (c *zfsCollector) updatePoolStats(ch chan<- prometheus.Metric) error { return errZFSNotAvailable } - err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) { + err = c.parseLinuxPoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) { ch <- c.constPoolObjsetMetric(poolName, datasetName, s, v) }) file.Close() @@ -220,7 +220,7 @@ func (c *zfsCollector) parsePoolProcfsFile(reader io.Reader, zpoolPath string, h return scanner.Err() } -func (c *zfsCollector) parsePoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error { +func (c *zfsCollector) parseLinuxPoolObjsetFile(reader io.Reader, zpoolPath string, handler func(string, string, zfsSysctl, uint64)) error { scanner := bufio.NewScanner(reader) parseLine := false diff --git a/collector/zfs_linux_test.go b/collector/zfs_linux_test.go index 69e4ed8fc1..f6ce8d9487 100644 --- a/collector/zfs_linux_test.go +++ b/collector/zfs_linux_test.go @@ -332,7 +332,7 @@ func TestZpoolObjsetParsing(t *testing.T) { t.Fatal(err) } - err = c.parsePoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) { + err = c.parseLinuxPoolObjsetFile(file, zpoolPath, func(poolName string, datasetName string, s zfsSysctl, v uint64) { if s != zfsSysctl("kstat.zfs.misc.objset.writes") { return }